

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta name="Description" content="scikit-learn: machine learning in Python">

  
  <title>7. Dataset loading utilities &mdash; scikit-learn 0.22 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/datasets/index.html" />

  
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script> 
</head>
<body>
<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../index.html">
        <img
          class="sk-brand-img"
          src="../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../modules/classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../getting_started.html">Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../tutorial/index.html">Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../glossary.html">Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../developers/index.html">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../faq.html">FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../related_projects.html">Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../roadmap.html">Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../about.html">About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../getting_started.html">Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../tutorial/index.html">Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../glossary.html">Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../developers/index.html">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../faq.html">FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../related_projects.html">Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../roadmap.html">Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../about.html">About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="sk-sidebar-toc-logo">
          <a href="../index.html">
            <img
              class="sk-brand-img"
              src="../_static/scikit-learn-logo-small.png"
              alt="logo"/>
          </a>
        </div>
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="../modules/preprocessing_targets.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="6.9. Transforming the prediction target (y)">Prev</a><a href="../user_guide.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="User Guide">Up</a>
            <a href="../modules/computing.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="8. Computing with scikit-learn">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 0.22</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
          <div class="sk-sidebar-toc">
            <ul>
<li><a class="reference internal" href="#">7. Dataset loading utilities</a><ul>
<li><a class="reference internal" href="#general-dataset-api">7.1. General dataset API</a></li>
<li><a class="reference internal" href="#toy-datasets">7.2. Toy datasets</a><ul>
<li><a class="reference internal" href="#boston-house-prices-dataset">7.2.1. Boston house prices dataset</a></li>
<li><a class="reference internal" href="#iris-plants-dataset">7.2.2. Iris plants dataset</a></li>
<li><a class="reference internal" href="#diabetes-dataset">7.2.3. Diabetes dataset</a></li>
<li><a class="reference internal" href="#optical-recognition-of-handwritten-digits-dataset">7.2.4. Optical recognition of handwritten digits dataset</a></li>
<li><a class="reference internal" href="#linnerrud-dataset">7.2.5. Linnerrud dataset</a></li>
<li><a class="reference internal" href="#wine-recognition-dataset">7.2.6. Wine recognition dataset</a></li>
<li><a class="reference internal" href="#breast-cancer-wisconsin-diagnostic-dataset">7.2.7. Breast cancer wisconsin (diagnostic) dataset</a></li>
</ul>
</li>
<li><a class="reference internal" href="#real-world-datasets">7.3. Real world datasets</a><ul>
<li><a class="reference internal" href="#the-olivetti-faces-dataset">7.3.1. The Olivetti faces dataset</a></li>
<li><a class="reference internal" href="#the-20-newsgroups-text-dataset">7.3.2. The 20 newsgroups text dataset</a><ul>
<li><a class="reference internal" href="#usage">7.3.2.1. Usage</a></li>
<li><a class="reference internal" href="#converting-text-to-vectors">7.3.2.2. Converting text to vectors</a></li>
<li><a class="reference internal" href="#filtering-text-for-more-realistic-training">7.3.2.3. Filtering text for more realistic training</a></li>
</ul>
</li>
<li><a class="reference internal" href="#the-labeled-faces-in-the-wild-face-recognition-dataset">7.3.3. The Labeled Faces in the Wild face recognition dataset</a><ul>
<li><a class="reference internal" href="#id5">7.3.3.1. Usage</a></li>
<li><a class="reference internal" href="#examples">7.3.3.2. Examples</a></li>
</ul>
</li>
<li><a class="reference internal" href="#forest-covertypes">7.3.4. Forest covertypes</a></li>
<li><a class="reference internal" href="#rcv1-dataset">7.3.5. RCV1 dataset</a></li>
<li><a class="reference internal" href="#kddcup-99-dataset">7.3.6. Kddcup 99 dataset</a></li>
<li><a class="reference internal" href="#california-housing-dataset">7.3.7. California Housing dataset</a></li>
</ul>
</li>
<li><a class="reference internal" href="#generated-datasets">7.4. Generated datasets</a><ul>
<li><a class="reference internal" href="#generators-for-classification-and-clustering">7.4.1. Generators for classification and clustering</a><ul>
<li><a class="reference internal" href="#single-label">7.4.1.1. Single label</a></li>
<li><a class="reference internal" href="#multilabel">7.4.1.2. Multilabel</a></li>
<li><a class="reference internal" href="#biclustering">7.4.1.3. Biclustering</a></li>
</ul>
</li>
<li><a class="reference internal" href="#generators-for-regression">7.4.2. Generators for regression</a></li>
<li><a class="reference internal" href="#generators-for-manifold-learning">7.4.3. Generators for manifold learning</a></li>
<li><a class="reference internal" href="#generators-for-decomposition">7.4.4. Generators for decomposition</a></li>
</ul>
</li>
<li><a class="reference internal" href="#loading-other-datasets">7.5. Loading other datasets</a><ul>
<li><a class="reference internal" href="#sample-images">7.5.1. Sample images</a></li>
<li><a class="reference internal" href="#datasets-in-svmlight-libsvm-format">7.5.2. Datasets in svmlight / libsvm format</a></li>
<li><a class="reference internal" href="#downloading-datasets-from-the-openml-org-repository">7.5.3. Downloading datasets from the openml.org repository</a><ul>
<li><a class="reference internal" href="#dataset-versions">7.5.3.1. Dataset Versions</a></li>
</ul>
</li>
<li><a class="reference internal" href="#loading-from-external-datasets">7.5.4. Loading from external datasets</a></li>
</ul>
</li>
</ul>
</li>
</ul>

          </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <div class="section" id="dataset-loading-utilities">
<span id="datasets"></span><h1>7. Dataset loading utilities<a class="headerlink" href="#dataset-loading-utilities" title="Permalink to this headline">¶</a></h1>
<p>The <code class="docutils literal notranslate"><span class="pre">sklearn.datasets</span></code> package embeds some small toy datasets
as introduced in the <a class="reference internal" href="../tutorial/basic/tutorial.html#loading-example-dataset"><span class="std std-ref">Getting Started</span></a> section.</p>
<p>This package also features helpers to fetch larger datasets commonly
used by the machine learning community to benchmark algorithms on data
that comes from the ‘real world’.</p>
<p>To evaluate the impact of the scale of the dataset (<code class="docutils literal notranslate"><span class="pre">n_samples</span></code> and
<code class="docutils literal notranslate"><span class="pre">n_features</span></code>) while controlling the statistical properties of the data
(typically the correlation and informativeness of the features), it is
also possible to generate synthetic data.</p>
<div class="section" id="general-dataset-api">
<h2>7.1. General dataset API<a class="headerlink" href="#general-dataset-api" title="Permalink to this headline">¶</a></h2>
<p>There are three main kinds of dataset interfaces that can be used to get
datasets depending on the desired type of dataset.</p>
<p><strong>The dataset loaders.</strong> They can be used to load small standard datasets,
described in the <a class="reference internal" href="#toy-datasets"><span class="std std-ref">Toy datasets</span></a> section.</p>
<p><strong>The dataset fetchers.</strong> They can be used to download and load larger datasets,
described in the <a class="reference internal" href="#real-world-datasets"><span class="std std-ref">Real world datasets</span></a> section.</p>
<p>Both loaders and fetchers functions return a dictionary-like object holding
at least two items: an array of shape <code class="docutils literal notranslate"><span class="pre">n_samples</span></code> * <code class="docutils literal notranslate"><span class="pre">n_features</span></code> with
key <code class="docutils literal notranslate"><span class="pre">data</span></code> (except for 20newsgroups) and a numpy array of
length <code class="docutils literal notranslate"><span class="pre">n_samples</span></code>, containing the target values, with key <code class="docutils literal notranslate"><span class="pre">target</span></code>.</p>
<p>It’s also possible for almost all of these function to constrain the output
to be a tuple containing only the data and the target, by setting the
<code class="docutils literal notranslate"><span class="pre">return_X_y</span></code> parameter to <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<p>The datasets also contain a full description in their <code class="docutils literal notranslate"><span class="pre">DESCR</span></code> attribute and
some contain <code class="docutils literal notranslate"><span class="pre">feature_names</span></code> and <code class="docutils literal notranslate"><span class="pre">target_names</span></code>. See the dataset
descriptions below for details.</p>
<p><strong>The dataset generation functions.</strong> They can be used to generate controlled
synthetic datasets, described in the <a class="reference internal" href="#sample-generators"><span class="std std-ref">Generated datasets</span></a> section.</p>
<p>These functions return a tuple <code class="docutils literal notranslate"><span class="pre">(X,</span> <span class="pre">y)</span></code> consisting of a <code class="docutils literal notranslate"><span class="pre">n_samples</span></code> *
<code class="docutils literal notranslate"><span class="pre">n_features</span></code> numpy array <code class="docutils literal notranslate"><span class="pre">X</span></code> and an array of length <code class="docutils literal notranslate"><span class="pre">n_samples</span></code>
containing the targets <code class="docutils literal notranslate"><span class="pre">y</span></code>.</p>
<p>In addition, there are also miscellaneous tools to load datasets of other
formats or from other locations, described in the <a class="reference internal" href="#loading-other-datasets"><span class="std std-ref">Loading other datasets</span></a>
section.</p>
</div>
<div class="section" id="toy-datasets">
<span id="id1"></span><h2>7.2. Toy datasets<a class="headerlink" href="#toy-datasets" title="Permalink to this headline">¶</a></h2>
<p>scikit-learn comes with a few small standard datasets that do not require to
download any file from some external website.</p>
<p>They can be loaded using the following functions:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_boston.html#sklearn.datasets.load_boston" title="sklearn.datasets.load_boston"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_boston</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the boston house-prices dataset (regression).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_iris.html#sklearn.datasets.load_iris" title="sklearn.datasets.load_iris"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_iris</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the iris dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_diabetes.html#sklearn.datasets.load_diabetes" title="sklearn.datasets.load_diabetes"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_diabetes</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the diabetes dataset (regression).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_digits.html#sklearn.datasets.load_digits" title="sklearn.datasets.load_digits"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_digits</span></code></a>([n_class, return_X_y])</p></td>
<td><p>Load and return the digits dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_linnerud.html#sklearn.datasets.load_linnerud" title="sklearn.datasets.load_linnerud"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_linnerud</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the linnerud dataset (multivariate regression).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_wine.html#sklearn.datasets.load_wine" title="sklearn.datasets.load_wine"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_wine</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the wine dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_breast_cancer.html#sklearn.datasets.load_breast_cancer" title="sklearn.datasets.load_breast_cancer"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_breast_cancer</span></code></a>([return_X_y])</p></td>
<td><p>Load and return the breast cancer wisconsin dataset (classification).</p></td>
</tr>
</tbody>
</table>
<p>These datasets are useful to quickly illustrate the behavior of the
various algorithms implemented in scikit-learn. They are however often too
small to be representative of real world machine learning tasks.</p>
<div class="section" id="boston-house-prices-dataset">
<span id="boston-dataset"></span><h3>7.2.1. Boston house prices dataset<a class="headerlink" href="#boston-house-prices-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>506</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.</p>
</dd>
<dt class="field-odd">Attribute Information (in order)</dt>
<dd class="field-odd"><ul class="simple">
<li><p>CRIM     per capita crime rate by town</p></li>
<li><p>ZN       proportion of residential land zoned for lots over 25,000 sq.ft.</p></li>
<li><p>INDUS    proportion of non-retail business acres per town</p></li>
<li><p>CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)</p></li>
<li><p>NOX      nitric oxides concentration (parts per 10 million)</p></li>
<li><p>RM       average number of rooms per dwelling</p></li>
<li><p>AGE      proportion of owner-occupied units built prior to 1940</p></li>
<li><p>DIS      weighted distances to five Boston employment centres</p></li>
<li><p>RAD      index of accessibility to radial highways</p></li>
<li><p>TAX      full-value property-tax rate per $10,000</p></li>
<li><p>PTRATIO  pupil-teacher ratio by town</p></li>
<li><p>B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town</p></li>
<li><p>LSTAT    % lower status of the population</p></li>
<li><p>MEDV     Median value of owner-occupied homes in $1000’s</p></li>
</ul>
</dd>
<dt class="field-even">Missing Attribute Values</dt>
<dd class="field-even"><p>None</p>
</dd>
<dt class="field-odd">Creator</dt>
<dd class="field-odd"><p>Harrison, D. and Rubinfeld, D.L.</p>
</dd>
</dl>
</div></blockquote>
<p>This is a copy of UCI ML housing dataset.
<a class="reference external" href="https://archive.ics.uci.edu/ml/machine-learning-databases/housing/">https://archive.ics.uci.edu/ml/machine-learning-databases/housing/</a></p>
<p>This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.</p>
<p>The Boston house-price data of Harrison, D. and Rubinfeld, D.L. ‘Hedonic
prices and the demand for clean air’, J. Environ. Economics &amp; Management,
vol.5, 81-102, 1978.   Used in Belsley, Kuh &amp; Welsch, ‘Regression diagnostics
…’, Wiley, 1980.   N.B. Various transformations are used in the table on
pages 244-261 of the latter.</p>
<p>The Boston house-price data has been used in many machine learning papers that address regression
problems.</p>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>Belsley, Kuh &amp; Welsch, ‘Regression diagnostics: Identifying Influential Data and Sources of Collinearity’, Wiley, 1980. 244-261.</p></li>
<li><p>Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.</p></li>
</ul>
</div>
</div>
<div class="section" id="iris-plants-dataset">
<span id="iris-dataset"></span><h3>7.2.2. Iris plants dataset<a class="headerlink" href="#iris-plants-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>150 (50 in each of three classes)</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>4 numeric, predictive attributes and the class</p>
</dd>
<dt class="field-odd">Attribute Information</dt>
<dd class="field-odd"><ul class="simple">
<li><p>sepal length in cm</p></li>
<li><p>sepal width in cm</p></li>
<li><p>petal length in cm</p></li>
<li><p>petal width in cm</p></li>
<li><dl class="simple">
<dt>class:</dt><dd><ul>
<li><p>Iris-Setosa</p></li>
<li><p>Iris-Versicolour</p></li>
<li><p>Iris-Virginica</p></li>
</ul>
</dd>
</dl>
</li>
</ul>
</dd>
<dt class="field-even">Summary Statistics</dt>
<dd class="field-even"><p></p></dd>
</dl>
<table class="docutils align-default">
<colgroup>
<col style="width: 26%" />
<col style="width: 7%" />
<col style="width: 7%" />
<col style="width: 13%" />
<col style="width: 9%" />
<col style="width: 37%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"></th>
<th class="head"></th>
<th class="head"></th>
<th class="head"></th>
<th class="head"></th>
<th class="head"></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>sepal length:</p></td>
<td><p>4.3</p></td>
<td><p>7.9</p></td>
<td><p>5.84</p></td>
<td><p>0.83</p></td>
<td><p>0.7826</p></td>
</tr>
<tr class="row-odd"><td><p>sepal width:</p></td>
<td><p>2.0</p></td>
<td><p>4.4</p></td>
<td><p>3.05</p></td>
<td><p>0.43</p></td>
<td><p>-0.4194</p></td>
</tr>
<tr class="row-even"><td><p>petal length:</p></td>
<td><p>1.0</p></td>
<td><p>6.9</p></td>
<td><p>3.76</p></td>
<td><p>1.76</p></td>
<td><p>0.9490  (high!)</p></td>
</tr>
<tr class="row-odd"><td><p>petal width:</p></td>
<td><p>0.1</p></td>
<td><p>2.5</p></td>
<td><p>1.20</p></td>
<td><p>0.76</p></td>
<td><p>0.9565  (high!)</p></td>
</tr>
</tbody>
</table>
<dl class="field-list simple">
<dt class="field-odd">Missing Attribute Values</dt>
<dd class="field-odd"><p>None</p>
</dd>
<dt class="field-even">Class Distribution</dt>
<dd class="field-even"><p>33.3% for each of 3 classes.</p>
</dd>
<dt class="field-odd">Creator</dt>
<dd class="field-odd"><p>R.A. Fisher</p>
</dd>
<dt class="field-even">Donor</dt>
<dd class="field-even"><p>Michael Marshall (<a class="reference external" href="mailto:MARSHALL%PLU&#37;&#52;&#48;io&#46;arc&#46;nasa&#46;gov">MARSHALL%PLU<span>&#64;</span>io<span>&#46;</span>arc<span>&#46;</span>nasa<span>&#46;</span>gov</a>)</p>
</dd>
<dt class="field-odd">Date</dt>
<dd class="field-odd"><p>July, 1988</p>
</dd>
</dl>
</div></blockquote>
<p>The famous Iris database, first used by Sir R.A. Fisher. The dataset is taken
from Fisher’s paper. Note that it’s the same as in R, but not as in the UCI
Machine Learning Repository, which has two wrong data points.</p>
<p>This is perhaps the best known database to be found in the
pattern recognition literature.  Fisher’s paper is a classic in the field and
is referenced frequently to this day.  (See Duda &amp; Hart, for example.)  The
data set contains 3 classes of 50 instances each, where each class refers to a
type of iris plant.  One class is linearly separable from the other 2; the
latter are NOT linearly separable from each other.</p>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>Fisher, R.A. “The use of multiple measurements in taxonomic problems”
Annual Eugenics, 7, Part II, 179-188 (1936); also in “Contributions to
Mathematical Statistics” (John Wiley, NY, 1950).</p></li>
<li><p>Duda, R.O., &amp; Hart, P.E. (1973) Pattern Classification and Scene Analysis.
(Q327.D83) John Wiley &amp; Sons.  ISBN 0-471-22361-1.  See page 218.</p></li>
<li><p>Dasarathy, B.V. (1980) “Nosing Around the Neighborhood: A New System
Structure and Classification Rule for Recognition in Partially Exposed
Environments”.  IEEE Transactions on Pattern Analysis and Machine
Intelligence, Vol. PAMI-2, No. 1, 67-71.</p></li>
<li><p>Gates, G.W. (1972) “The Reduced Nearest Neighbor Rule”.  IEEE Transactions
on Information Theory, May 1972, 431-433.</p></li>
<li><p>See also: 1988 MLC Proceedings, 54-64.  Cheeseman et al”s AUTOCLASS II
conceptual clustering system finds 3 classes in the data.</p></li>
<li><p>Many, many more …</p></li>
</ul>
</div>
</div>
<div class="section" id="diabetes-dataset">
<span id="id2"></span><h3>7.2.3. Diabetes dataset<a class="headerlink" href="#diabetes-dataset" title="Permalink to this headline">¶</a></h3>
<p>Ten baseline variables, age, sex, body mass index, average blood
pressure, and six blood serum measurements were obtained for each of n =
442 diabetes patients, as well as the response of interest, a
quantitative measure of disease progression one year after baseline.</p>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>442</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>First 10 columns are numeric predictive values</p>
</dd>
<dt class="field-odd">Target</dt>
<dd class="field-odd"><p>Column 11 is a quantitative measure of disease progression one year after baseline</p>
</dd>
<dt class="field-even">Attribute Information</dt>
<dd class="field-even"><ul class="simple">
<li><p>Age</p></li>
<li><p>Sex</p></li>
<li><p>Body mass index</p></li>
<li><p>Average blood pressure</p></li>
<li><p>S1</p></li>
<li><p>S2</p></li>
<li><p>S3</p></li>
<li><p>S4</p></li>
<li><p>S5</p></li>
<li><p>S6</p></li>
</ul>
</dd>
</dl>
</div></blockquote>
<p>Note: Each of these 10 feature variables have been mean centered and scaled by the standard deviation times <code class="docutils literal notranslate"><span class="pre">n_samples</span></code> (i.e. the sum of squares of each column totals 1).</p>
<p>Source URL:
<a class="reference external" href="https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html">https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html</a></p>
<p>For more information see:
Bradley Efron, Trevor Hastie, Iain Johnstone and Robert Tibshirani (2004) “Least Angle Regression,” Annals of Statistics (with discussion), 407-499.
(<a class="reference external" href="https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf">https://web.stanford.edu/~hastie/Papers/LARS/LeastAngle_2002.pdf</a>)</p>
</div>
<div class="section" id="optical-recognition-of-handwritten-digits-dataset">
<span id="digits-dataset"></span><h3>7.2.4. Optical recognition of handwritten digits dataset<a class="headerlink" href="#optical-recognition-of-handwritten-digits-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>5620</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>64</p>
</dd>
<dt class="field-odd">Attribute Information</dt>
<dd class="field-odd"><p>8x8 image of integer pixels in the range 0..16.</p>
</dd>
<dt class="field-even">Missing Attribute Values</dt>
<dd class="field-even"><p>None</p>
</dd>
<dt class="field-odd">Creator</dt>
<dd class="field-odd"><ol class="upperalpha simple" start="5">
<li><p>Alpaydin (alpaydin ‘&#64;’ boun.edu.tr)</p></li>
</ol>
</dd>
<dt class="field-even">Date</dt>
<dd class="field-even"><p>July; 1998</p>
</dd>
</dl>
</div></blockquote>
<p>This is a copy of the test set of the UCI ML hand-written digits datasets
<a class="reference external" href="https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits">https://archive.ics.uci.edu/ml/datasets/Optical+Recognition+of+Handwritten+Digits</a></p>
<p>The data set contains images of hand-written digits: 10 classes where
each class refers to a digit.</p>
<p>Preprocessing programs made available by NIST were used to extract
normalized bitmaps of handwritten digits from a preprinted form. From a
total of 43 people, 30 contributed to the training set and different 13
to the test set. 32x32 bitmaps are divided into nonoverlapping blocks of
4x4 and the number of on pixels are counted in each block. This generates
an input matrix of 8x8 where each element is an integer in the range
0..16. This reduces dimensionality and gives invariance to small
distortions.</p>
<p>For info on NIST preprocessing routines, see M. D. Garris, J. L. Blue, G.
T. Candela, D. L. Dimmick, J. Geist, P. J. Grother, S. A. Janet, and C.
L. Wilson, NIST Form-Based Handprint Recognition System, NISTIR 5469,
1994.</p>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>C. Kaynak (1995) Methods of Combining Multiple Classifiers and Their
Applications to Handwritten Digit Recognition, MSc Thesis, Institute of
Graduate Studies in Science and Engineering, Bogazici University.</p></li>
<li><ol class="upperalpha simple" start="5">
<li><p>Alpaydin, C. Kaynak (1998) Cascading Classifiers, Kybernetika.</p></li>
</ol>
</li>
<li><p>Ken Tang and Ponnuthurai N. Suganthan and Xi Yao and A. Kai Qin.
Linear dimensionalityreduction using relevance weighted LDA. School of
Electrical and Electronic Engineering Nanyang Technological University.
2005.</p></li>
<li><p>Claudio Gentile. A New Approximate Maximal Margin Classification
Algorithm. NIPS. 2000.</p></li>
</ul>
</div>
</div>
<div class="section" id="linnerrud-dataset">
<span id="id3"></span><h3>7.2.5. Linnerrud dataset<a class="headerlink" href="#linnerrud-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>20</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>3</p>
</dd>
<dt class="field-odd">Missing Attribute Values</dt>
<dd class="field-odd"><p>None</p>
</dd>
</dl>
</div></blockquote>
<p>The Linnerud dataset constains two small dataset:</p>
<ul class="simple">
<li><dl class="simple">
<dt><em>physiological</em> - CSV containing 20 observations on 3 exercise variables:</dt><dd><p>Weight, Waist and Pulse.</p>
</dd>
</dl>
</li>
<li><dl class="simple">
<dt><em>exercise</em> - CSV containing 20 observations on 3 physiological variables:</dt><dd><p>Chins, Situps and Jumps.</p>
</dd>
</dl>
</li>
</ul>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>Tenenhaus, M. (1998). La regression PLS: theorie et pratique. Paris: Editions Technic.</p></li>
</ul>
</div>
</div>
<div class="section" id="wine-recognition-dataset">
<span id="wine-dataset"></span><h3>7.2.6. Wine recognition dataset<a class="headerlink" href="#wine-recognition-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>178 (50 in each of three classes)</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>13 numeric, predictive attributes and the class</p>
</dd>
<dt class="field-odd">Attribute Information</dt>
<dd class="field-odd"><ul class="simple">
<li><p>Alcohol</p></li>
<li><p>Malic acid</p></li>
<li><p>Ash</p></li>
<li><p>Alcalinity of ash</p></li>
<li><p>Magnesium</p></li>
<li><p>Total phenols</p></li>
<li><p>Flavanoids</p></li>
<li><p>Nonflavanoid phenols</p></li>
<li><p>Proanthocyanins</p></li>
<li><p>Color intensity</p></li>
<li><p>Hue</p></li>
<li><p>OD280/OD315 of diluted wines</p></li>
<li><p>Proline</p></li>
</ul>
</dd>
</dl>
<ul class="simple">
<li><dl class="simple">
<dt>class:</dt><dd><ul>
<li><p>class_0</p></li>
<li><p>class_1</p></li>
<li><p>class_2</p></li>
</ul>
</dd>
</dl>
</li>
</ul>
<dl class="field-list simple">
<dt class="field-odd">Summary Statistics</dt>
<dd class="field-odd"><p></p></dd>
</dl>
<table class="docutils align-default">
<colgroup>
<col style="width: 58%" />
<col style="width: 8%" />
<col style="width: 10%" />
<col style="width: 14%" />
<col style="width: 10%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"></th>
<th class="head"></th>
<th class="head"></th>
<th class="head"></th>
<th class="head"></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>Alcohol:</p></td>
<td><p>11.0</p></td>
<td><p>14.8</p></td>
<td><p>13.0</p></td>
<td><p>0.8</p></td>
</tr>
<tr class="row-odd"><td><p>Malic Acid:</p></td>
<td><p>0.74</p></td>
<td><p>5.80</p></td>
<td><p>2.34</p></td>
<td><p>1.12</p></td>
</tr>
<tr class="row-even"><td><p>Ash:</p></td>
<td><p>1.36</p></td>
<td><p>3.23</p></td>
<td><p>2.36</p></td>
<td><p>0.27</p></td>
</tr>
<tr class="row-odd"><td><p>Alcalinity of Ash:</p></td>
<td><p>10.6</p></td>
<td><p>30.0</p></td>
<td><p>19.5</p></td>
<td><p>3.3</p></td>
</tr>
<tr class="row-even"><td><p>Magnesium:</p></td>
<td><p>70.0</p></td>
<td><p>162.0</p></td>
<td><p>99.7</p></td>
<td><p>14.3</p></td>
</tr>
<tr class="row-odd"><td><p>Total Phenols:</p></td>
<td><p>0.98</p></td>
<td><p>3.88</p></td>
<td><p>2.29</p></td>
<td><p>0.63</p></td>
</tr>
<tr class="row-even"><td><p>Flavanoids:</p></td>
<td><p>0.34</p></td>
<td><p>5.08</p></td>
<td><p>2.03</p></td>
<td><p>1.00</p></td>
</tr>
<tr class="row-odd"><td><p>Nonflavanoid Phenols:</p></td>
<td><p>0.13</p></td>
<td><p>0.66</p></td>
<td><p>0.36</p></td>
<td><p>0.12</p></td>
</tr>
<tr class="row-even"><td><p>Proanthocyanins:</p></td>
<td><p>0.41</p></td>
<td><p>3.58</p></td>
<td><p>1.59</p></td>
<td><p>0.57</p></td>
</tr>
<tr class="row-odd"><td><p>Colour Intensity:</p></td>
<td><p>1.3</p></td>
<td><p>13.0</p></td>
<td><p>5.1</p></td>
<td><p>2.3</p></td>
</tr>
<tr class="row-even"><td><p>Hue:</p></td>
<td><p>0.48</p></td>
<td><p>1.71</p></td>
<td><p>0.96</p></td>
<td><p>0.23</p></td>
</tr>
<tr class="row-odd"><td><p>OD280/OD315 of diluted wines:</p></td>
<td><p>1.27</p></td>
<td><p>4.00</p></td>
<td><p>2.61</p></td>
<td><p>0.71</p></td>
</tr>
<tr class="row-even"><td><p>Proline:</p></td>
<td><p>278</p></td>
<td><p>1680</p></td>
<td><p>746</p></td>
<td><p>315</p></td>
</tr>
</tbody>
</table>
<dl class="field-list simple">
<dt class="field-odd">Missing Attribute Values</dt>
<dd class="field-odd"><p>None</p>
</dd>
<dt class="field-even">Class Distribution</dt>
<dd class="field-even"><p>class_0 (59), class_1 (71), class_2 (48)</p>
</dd>
<dt class="field-odd">Creator</dt>
<dd class="field-odd"><p>R.A. Fisher</p>
</dd>
<dt class="field-even">Donor</dt>
<dd class="field-even"><p>Michael Marshall (<a class="reference external" href="mailto:MARSHALL%PLU&#37;&#52;&#48;io&#46;arc&#46;nasa&#46;gov">MARSHALL%PLU<span>&#64;</span>io<span>&#46;</span>arc<span>&#46;</span>nasa<span>&#46;</span>gov</a>)</p>
</dd>
<dt class="field-odd">Date</dt>
<dd class="field-odd"><p>July, 1988</p>
</dd>
</dl>
</div></blockquote>
<p>This is a copy of UCI ML Wine recognition datasets.
<a class="reference external" href="https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data">https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data</a></p>
<p>The data is the results of a chemical analysis of wines grown in the same
region in Italy by three different cultivators. There are thirteen different
measurements taken for different constituents found in the three types of
wine.</p>
<p>Original Owners:</p>
<p>Forina, M. et al, PARVUS -
An Extendible Package for Data Exploration, Classification and Correlation.
Institute of Pharmaceutical and Food Analysis and Technologies,
Via Brigata Salerno, 16147 Genoa, Italy.</p>
<p>Citation:</p>
<p>Lichman, M. (2013). UCI Machine Learning Repository
[<a class="reference external" href="https://archive.ics.uci.edu/ml">https://archive.ics.uci.edu/ml</a>]. Irvine, CA: University of California,
School of Information and Computer Science.</p>
<div class="topic">
<p class="topic-title">References</p>
<p>(1) S. Aeberhard, D. Coomans and O. de Vel,
Comparison of Classifiers in High Dimensional Settings,
Tech. Rep. no. 92-02, (1992), Dept. of Computer Science and Dept. of
Mathematics and Statistics, James Cook University of North Queensland.
(Also submitted to Technometrics).</p>
<p>The data was used with many others for comparing various
classifiers. The classes are separable, though only RDA
has achieved 100% correct classification.
(RDA : 100%, QDA 99.4%, LDA 98.9%, 1NN 96.1% (z-transformed data))
(All results using the leave-one-out technique)</p>
<p>(2) S. Aeberhard, D. Coomans and O. de Vel,
“THE CLASSIFICATION PERFORMANCE OF RDA”
Tech. Rep. no. 92-01, (1992), Dept. of Computer Science and Dept. of
Mathematics and Statistics, James Cook University of North Queensland.
(Also submitted to Journal of Chemometrics).</p>
</div>
</div>
<div class="section" id="breast-cancer-wisconsin-diagnostic-dataset">
<span id="breast-cancer-dataset"></span><h3>7.2.7. Breast cancer wisconsin (diagnostic) dataset<a class="headerlink" href="#breast-cancer-wisconsin-diagnostic-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>569</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>30 numeric, predictive attributes and the class</p>
</dd>
<dt class="field-odd">Attribute Information</dt>
<dd class="field-odd"><ul class="simple">
<li><p>radius (mean of distances from center to points on the perimeter)</p></li>
<li><p>texture (standard deviation of gray-scale values)</p></li>
<li><p>perimeter</p></li>
<li><p>area</p></li>
<li><p>smoothness (local variation in radius lengths)</p></li>
<li><p>compactness (perimeter^2 / area - 1.0)</p></li>
<li><p>concavity (severity of concave portions of the contour)</p></li>
<li><p>concave points (number of concave portions of the contour)</p></li>
<li><p>symmetry</p></li>
<li><p>fractal dimension (“coastline approximation” - 1)</p></li>
</ul>
<p>The mean, standard error, and “worst” or largest (mean of the three
largest values) of these features were computed for each image,
resulting in 30 features.  For instance, field 3 is Mean Radius, field
13 is Radius SE, field 23 is Worst Radius.</p>
<ul class="simple">
<li><dl class="simple">
<dt>class:</dt><dd><ul>
<li><p>WDBC-Malignant</p></li>
<li><p>WDBC-Benign</p></li>
</ul>
</dd>
</dl>
</li>
</ul>
</dd>
<dt class="field-even">Summary Statistics</dt>
<dd class="field-even"><p></p></dd>
</dl>
<table class="docutils align-default">
<colgroup>
<col style="width: 76%" />
<col style="width: 12%" />
<col style="width: 12%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"></th>
<th class="head"></th>
<th class="head"></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>radius (mean):</p></td>
<td><p>6.981</p></td>
<td><p>28.11</p></td>
</tr>
<tr class="row-odd"><td><p>texture (mean):</p></td>
<td><p>9.71</p></td>
<td><p>39.28</p></td>
</tr>
<tr class="row-even"><td><p>perimeter (mean):</p></td>
<td><p>43.79</p></td>
<td><p>188.5</p></td>
</tr>
<tr class="row-odd"><td><p>area (mean):</p></td>
<td><p>143.5</p></td>
<td><p>2501.0</p></td>
</tr>
<tr class="row-even"><td><p>smoothness (mean):</p></td>
<td><p>0.053</p></td>
<td><p>0.163</p></td>
</tr>
<tr class="row-odd"><td><p>compactness (mean):</p></td>
<td><p>0.019</p></td>
<td><p>0.345</p></td>
</tr>
<tr class="row-even"><td><p>concavity (mean):</p></td>
<td><p>0.0</p></td>
<td><p>0.427</p></td>
</tr>
<tr class="row-odd"><td><p>concave points (mean):</p></td>
<td><p>0.0</p></td>
<td><p>0.201</p></td>
</tr>
<tr class="row-even"><td><p>symmetry (mean):</p></td>
<td><p>0.106</p></td>
<td><p>0.304</p></td>
</tr>
<tr class="row-odd"><td><p>fractal dimension (mean):</p></td>
<td><p>0.05</p></td>
<td><p>0.097</p></td>
</tr>
<tr class="row-even"><td><p>radius (standard error):</p></td>
<td><p>0.112</p></td>
<td><p>2.873</p></td>
</tr>
<tr class="row-odd"><td><p>texture (standard error):</p></td>
<td><p>0.36</p></td>
<td><p>4.885</p></td>
</tr>
<tr class="row-even"><td><p>perimeter (standard error):</p></td>
<td><p>0.757</p></td>
<td><p>21.98</p></td>
</tr>
<tr class="row-odd"><td><p>area (standard error):</p></td>
<td><p>6.802</p></td>
<td><p>542.2</p></td>
</tr>
<tr class="row-even"><td><p>smoothness (standard error):</p></td>
<td><p>0.002</p></td>
<td><p>0.031</p></td>
</tr>
<tr class="row-odd"><td><p>compactness (standard error):</p></td>
<td><p>0.002</p></td>
<td><p>0.135</p></td>
</tr>
<tr class="row-even"><td><p>concavity (standard error):</p></td>
<td><p>0.0</p></td>
<td><p>0.396</p></td>
</tr>
<tr class="row-odd"><td><p>concave points (standard error):</p></td>
<td><p>0.0</p></td>
<td><p>0.053</p></td>
</tr>
<tr class="row-even"><td><p>symmetry (standard error):</p></td>
<td><p>0.008</p></td>
<td><p>0.079</p></td>
</tr>
<tr class="row-odd"><td><p>fractal dimension (standard error):</p></td>
<td><p>0.001</p></td>
<td><p>0.03</p></td>
</tr>
<tr class="row-even"><td><p>radius (worst):</p></td>
<td><p>7.93</p></td>
<td><p>36.04</p></td>
</tr>
<tr class="row-odd"><td><p>texture (worst):</p></td>
<td><p>12.02</p></td>
<td><p>49.54</p></td>
</tr>
<tr class="row-even"><td><p>perimeter (worst):</p></td>
<td><p>50.41</p></td>
<td><p>251.2</p></td>
</tr>
<tr class="row-odd"><td><p>area (worst):</p></td>
<td><p>185.2</p></td>
<td><p>4254.0</p></td>
</tr>
<tr class="row-even"><td><p>smoothness (worst):</p></td>
<td><p>0.071</p></td>
<td><p>0.223</p></td>
</tr>
<tr class="row-odd"><td><p>compactness (worst):</p></td>
<td><p>0.027</p></td>
<td><p>1.058</p></td>
</tr>
<tr class="row-even"><td><p>concavity (worst):</p></td>
<td><p>0.0</p></td>
<td><p>1.252</p></td>
</tr>
<tr class="row-odd"><td><p>concave points (worst):</p></td>
<td><p>0.0</p></td>
<td><p>0.291</p></td>
</tr>
<tr class="row-even"><td><p>symmetry (worst):</p></td>
<td><p>0.156</p></td>
<td><p>0.664</p></td>
</tr>
<tr class="row-odd"><td><p>fractal dimension (worst):</p></td>
<td><p>0.055</p></td>
<td><p>0.208</p></td>
</tr>
</tbody>
</table>
<dl class="field-list simple">
<dt class="field-odd">Missing Attribute Values</dt>
<dd class="field-odd"><p>None</p>
</dd>
<dt class="field-even">Class Distribution</dt>
<dd class="field-even"><p>212 - Malignant, 357 - Benign</p>
</dd>
<dt class="field-odd">Creator</dt>
<dd class="field-odd"><p>Dr. William H. Wolberg, W. Nick Street, Olvi L. Mangasarian</p>
</dd>
<dt class="field-even">Donor</dt>
<dd class="field-even"><p>Nick Street</p>
</dd>
<dt class="field-odd">Date</dt>
<dd class="field-odd"><p>November, 1995</p>
</dd>
</dl>
</div></blockquote>
<p>This is a copy of UCI ML Breast Cancer Wisconsin (Diagnostic) datasets.
<a class="reference external" href="https://goo.gl/U2Uwz2">https://goo.gl/U2Uwz2</a></p>
<p>Features are computed from a digitized image of a fine needle
aspirate (FNA) of a breast mass.  They describe
characteristics of the cell nuclei present in the image.</p>
<p>Separating plane described above was obtained using
Multisurface Method-Tree (MSM-T) [K. P. Bennett, “Decision Tree
Construction Via Linear Programming.” Proceedings of the 4th
Midwest Artificial Intelligence and Cognitive Science Society,
pp. 97-101, 1992], a classification method which uses linear
programming to construct a decision tree.  Relevant features
were selected using an exhaustive search in the space of 1-4
features and 1-3 separating planes.</p>
<p>The actual linear program used to obtain the separating plane
in the 3-dimensional space is that described in:
[K. P. Bennett and O. L. Mangasarian: “Robust Linear
Programming Discrimination of Two Linearly Inseparable Sets”,
Optimization Methods and Software 1, 1992, 23-34].</p>
<p>This database is also available through the UW CS ftp server:</p>
<p>ftp ftp.cs.wisc.edu
cd math-prog/cpo-dataset/machine-learn/WDBC/</p>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>W.N. Street, W.H. Wolberg and O.L. Mangasarian. Nuclear feature extraction
for breast tumor diagnosis. IS&amp;T/SPIE 1993 International Symposium on
Electronic Imaging: Science and Technology, volume 1905, pages 861-870,
San Jose, CA, 1993.</p></li>
<li><p>O.L. Mangasarian, W.N. Street and W.H. Wolberg. Breast cancer diagnosis and
prognosis via linear programming. Operations Research, 43(4), pages 570-577,
July-August 1995.</p></li>
<li><p>W.H. Wolberg, W.N. Street, and O.L. Mangasarian. Machine learning techniques
to diagnose breast cancer from fine-needle aspirates. Cancer Letters 77 (1994)
163-171.</p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="real-world-datasets">
<span id="id4"></span><h2>7.3. Real world datasets<a class="headerlink" href="#real-world-datasets" title="Permalink to this headline">¶</a></h2>
<p>scikit-learn provides tools to load larger datasets, downloading them if
necessary.</p>
<p>They can be loaded using the following functions:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_olivetti_faces.html#sklearn.datasets.fetch_olivetti_faces" title="sklearn.datasets.fetch_olivetti_faces"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_olivetti_faces</span></code></a>([data_home, shuffle, …])</p></td>
<td><p>Load the Olivetti faces data-set from AT&amp;T (classification).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups.html#sklearn.datasets.fetch_20newsgroups" title="sklearn.datasets.fetch_20newsgroups"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_20newsgroups</span></code></a>([data_home, subset, …])</p></td>
<td><p>Load the filenames and data from the 20 newsgroups dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups_vectorized.html#sklearn.datasets.fetch_20newsgroups_vectorized" title="sklearn.datasets.fetch_20newsgroups_vectorized"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_20newsgroups_vectorized</span></code></a>([subset, …])</p></td>
<td><p>Load the 20 newsgroups dataset and vectorize it into token counts (classification).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_lfw_people.html#sklearn.datasets.fetch_lfw_people" title="sklearn.datasets.fetch_lfw_people"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_lfw_people</span></code></a>([data_home, funneled, …])</p></td>
<td><p>Load the Labeled Faces in the Wild (LFW) people dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_lfw_pairs.html#sklearn.datasets.fetch_lfw_pairs" title="sklearn.datasets.fetch_lfw_pairs"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_lfw_pairs</span></code></a>([subset, data_home, …])</p></td>
<td><p>Load the Labeled Faces in the Wild (LFW) pairs dataset (classification).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_covtype.html#sklearn.datasets.fetch_covtype" title="sklearn.datasets.fetch_covtype"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_covtype</span></code></a>([data_home, …])</p></td>
<td><p>Load the covertype dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_rcv1.html#sklearn.datasets.fetch_rcv1" title="sklearn.datasets.fetch_rcv1"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_rcv1</span></code></a>([data_home, subset, …])</p></td>
<td><p>Load the RCV1 multilabel dataset (classification).</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_kddcup99.html#sklearn.datasets.fetch_kddcup99" title="sklearn.datasets.fetch_kddcup99"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_kddcup99</span></code></a>([subset, data_home, shuffle, …])</p></td>
<td><p>Load the kddcup99 dataset (classification).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_california_housing.html#sklearn.datasets.fetch_california_housing" title="sklearn.datasets.fetch_california_housing"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fetch_california_housing</span></code></a>([data_home, …])</p></td>
<td><p>Load the California housing dataset (regression).</p></td>
</tr>
</tbody>
</table>
<div class="section" id="the-olivetti-faces-dataset">
<span id="olivetti-faces-dataset"></span><h3>7.3.1. The Olivetti faces dataset<a class="headerlink" href="#the-olivetti-faces-dataset" title="Permalink to this headline">¶</a></h3>
<p><a class="reference external" href="http://www.cl.cam.ac.uk/research/dtg/attarchive/facedatabase.html">This dataset contains a set of face images</a> taken between April 1992 and
April 1994 at AT&amp;T Laboratories Cambridge. The
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_olivetti_faces.html#sklearn.datasets.fetch_olivetti_faces" title="sklearn.datasets.fetch_olivetti_faces"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_olivetti_faces</span></code></a> function is the data
fetching / caching function that downloads the data
archive from AT&amp;T.</p>
<p>As described on the original website:</p>
<blockquote>
<div><p>There are ten different images of each of 40 distinct subjects. For some
subjects, the images were taken at different times, varying the lighting,
facial expressions (open / closed eyes, smiling / not smiling) and facial
details (glasses / no glasses). All the images were taken against a dark
homogeneous background with the subjects in an upright, frontal position
(with tolerance for some side movement).</p>
</div></blockquote>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 45%" />
<col style="width: 55%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Classes</p></td>
<td><p>40</p></td>
</tr>
<tr class="row-even"><td><p>Samples total</p></td>
<td><p>400</p></td>
</tr>
<tr class="row-odd"><td><p>Dimensionality</p></td>
<td><p>4096</p></td>
</tr>
<tr class="row-even"><td><p>Features</p></td>
<td><p>real, between 0 and 1</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<p>The image is quantized to 256 grey levels and stored as unsigned 8-bit
integers; the loader will convert these to floating point values on the
interval [0, 1], which are easier to work with for many algorithms.</p>
<p>The “target” for this database is an integer from 0 to 39 indicating the
identity of the person pictured; however, with only 10 examples per class, this
relatively small dataset is more interesting from an unsupervised or
semi-supervised perspective.</p>
<p>The original dataset consisted of 92 x 112, while the version available here
consists of 64x64 images.</p>
<p>When using these images, please give credit to AT&amp;T Laboratories Cambridge.</p>
</div>
<div class="section" id="the-20-newsgroups-text-dataset">
<span id="newsgroups-dataset"></span><h3>7.3.2. The 20 newsgroups text dataset<a class="headerlink" href="#the-20-newsgroups-text-dataset" title="Permalink to this headline">¶</a></h3>
<p>The 20 newsgroups dataset comprises around 18000 newsgroups posts on
20 topics split in two subsets: one for training (or development)
and the other one for testing (or for performance evaluation). The split
between the train and test set is based upon a messages posted before
and after a specific date.</p>
<p>This module contains two loaders. The first one,
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups.html#sklearn.datasets.fetch_20newsgroups" title="sklearn.datasets.fetch_20newsgroups"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_20newsgroups</span></code></a>,
returns a list of the raw texts that can be fed to text feature
extractors such as <a class="reference internal" href="../modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer" title="sklearn.feature_extraction.text.CountVectorizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn.feature_extraction.text.CountVectorizer</span></code></a>
with custom parameters so as to extract feature vectors.
The second one, <a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups_vectorized.html#sklearn.datasets.fetch_20newsgroups_vectorized" title="sklearn.datasets.fetch_20newsgroups_vectorized"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_20newsgroups_vectorized</span></code></a>,
returns ready-to-use features, i.e., it is not necessary to use a feature
extractor.</p>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 63%" />
<col style="width: 37%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Classes</p></td>
<td><p>20</p></td>
</tr>
<tr class="row-even"><td><p>Samples total</p></td>
<td><p>18846</p></td>
</tr>
<tr class="row-odd"><td><p>Dimensionality</p></td>
<td><p>1</p></td>
</tr>
<tr class="row-even"><td><p>Features</p></td>
<td><p>text</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<div class="section" id="usage">
<h4>7.3.2.1. Usage<a class="headerlink" href="#usage" title="Permalink to this headline">¶</a></h4>
<p>The <a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups.html#sklearn.datasets.fetch_20newsgroups" title="sklearn.datasets.fetch_20newsgroups"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_20newsgroups</span></code></a> function is a data
fetching / caching functions that downloads the data archive from
the original <a class="reference external" href="http://people.csail.mit.edu/jrennie/20Newsgroups/">20 newsgroups website</a>, extracts the archive contents
in the <code class="docutils literal notranslate"><span class="pre">~/scikit_learn_data/20news_home</span></code> folder and calls the
<a class="reference internal" href="../modules/generated/sklearn.datasets.load_files.html#sklearn.datasets.load_files" title="sklearn.datasets.load_files"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.load_files</span></code></a> on either the training or
testing set folder, or both of them:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_20newsgroups</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pprint</span> <span class="kn">import</span> <span class="n">pprint</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pprint</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">))</span>
<span class="go">[&#39;alt.atheism&#39;,</span>
<span class="go"> &#39;comp.graphics&#39;,</span>
<span class="go"> &#39;comp.os.ms-windows.misc&#39;,</span>
<span class="go"> &#39;comp.sys.ibm.pc.hardware&#39;,</span>
<span class="go"> &#39;comp.sys.mac.hardware&#39;,</span>
<span class="go"> &#39;comp.windows.x&#39;,</span>
<span class="go"> &#39;misc.forsale&#39;,</span>
<span class="go"> &#39;rec.autos&#39;,</span>
<span class="go"> &#39;rec.motorcycles&#39;,</span>
<span class="go"> &#39;rec.sport.baseball&#39;,</span>
<span class="go"> &#39;rec.sport.hockey&#39;,</span>
<span class="go"> &#39;sci.crypt&#39;,</span>
<span class="go"> &#39;sci.electronics&#39;,</span>
<span class="go"> &#39;sci.med&#39;,</span>
<span class="go"> &#39;sci.space&#39;,</span>
<span class="go"> &#39;soc.religion.christian&#39;,</span>
<span class="go"> &#39;talk.politics.guns&#39;,</span>
<span class="go"> &#39;talk.politics.mideast&#39;,</span>
<span class="go"> &#39;talk.politics.misc&#39;,</span>
<span class="go"> &#39;talk.religion.misc&#39;]</span>
</pre></div>
</div>
<p>The real data lies in the <code class="docutils literal notranslate"><span class="pre">filenames</span></code> and <code class="docutils literal notranslate"><span class="pre">target</span></code> attributes. The target
attribute is the integer index of the category:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">filenames</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(11314,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(11314,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">10</span><span class="p">]</span>
<span class="go">array([ 7,  4,  4,  1, 14, 16, 13,  3,  2,  4])</span>
</pre></div>
</div>
<p>It is possible to load only a sub-selection of the categories by passing the
list of the categories to load to the
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups.html#sklearn.datasets.fetch_20newsgroups" title="sklearn.datasets.fetch_20newsgroups"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_20newsgroups</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">cats</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;alt.atheism&#39;</span><span class="p">,</span> <span class="s1">&#39;sci.space&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">,</span> <span class="n">categories</span><span class="o">=</span><span class="n">cats</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="nb">list</span><span class="p">(</span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">)</span>
<span class="go">[&#39;alt.atheism&#39;, &#39;sci.space&#39;]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">filenames</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1073,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1073,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">10</span><span class="p">]</span>
<span class="go">array([0, 1, 1, 1, 0, 1, 1, 0, 0, 0])</span>
</pre></div>
</div>
</div>
<div class="section" id="converting-text-to-vectors">
<h4>7.3.2.2. Converting text to vectors<a class="headerlink" href="#converting-text-to-vectors" title="Permalink to this headline">¶</a></h4>
<p>In order to feed predictive or clustering models with the text data,
one first need to turn the text into vectors of numerical values suitable
for statistical analysis. This can be achieved with the utilities of the
<code class="docutils literal notranslate"><span class="pre">sklearn.feature_extraction.text</span></code> as demonstrated in the following
example that extract <a class="reference external" href="https://en.wikipedia.org/wiki/Tf-idf">TF-IDF</a> vectors of unigram tokens
from a subset of 20news:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.feature_extraction.text</span> <span class="kn">import</span> <span class="n">TfidfVectorizer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">categories</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;alt.atheism&#39;</span><span class="p">,</span> <span class="s1">&#39;talk.religion.misc&#39;</span><span class="p">,</span>
<span class="gp">... </span>              <span class="s1">&#39;comp.graphics&#39;</span><span class="p">,</span> <span class="s1">&#39;sci.space&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">,</span>
<span class="gp">... </span>                                      <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectorizer</span> <span class="o">=</span> <span class="n">TfidfVectorizer</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectors</span> <span class="o">=</span> <span class="n">vectorizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectors</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2034, 34118)</span>
</pre></div>
</div>
<p>The extracted TF-IDF vectors are very sparse, with an average of 159 non-zero
components by sample in a more than 30000-dimensional space
(less than .5% non-zero features):</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">vectors</span><span class="o">.</span><span class="n">nnz</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">vectors</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="go">159.01327...</span>
</pre></div>
</div>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_20newsgroups_vectorized.html#sklearn.datasets.fetch_20newsgroups_vectorized" title="sklearn.datasets.fetch_20newsgroups_vectorized"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_20newsgroups_vectorized</span></code></a> is a function which
returns ready-to-use token counts features instead of file names.</p>
</div>
<div class="section" id="filtering-text-for-more-realistic-training">
<h4>7.3.2.3. Filtering text for more realistic training<a class="headerlink" href="#filtering-text-for-more-realistic-training" title="Permalink to this headline">¶</a></h4>
<p>It is easy for a classifier to overfit on particular things that appear in the
20 Newsgroups data, such as newsgroup headers. Many classifiers achieve very
high F-scores, but their results would not generalize to other documents that
aren’t from this window of time.</p>
<p>For example, let’s look at the results of a multinomial Naive Bayes classifier,
which is fast to train and achieves a decent F-score:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.naive_bayes</span> <span class="kn">import</span> <span class="n">MultinomialNB</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">metrics</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_test</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;test&#39;</span><span class="p">,</span>
<span class="gp">... </span>                                     <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectors_test</span> <span class="o">=</span> <span class="n">vectorizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">newsgroups_test</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">MultinomialNB</span><span class="p">(</span><span class="n">alpha</span><span class="o">=.</span><span class="mi">01</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">vectors</span><span class="p">,</span> <span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">pred</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">vectors_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">f1_score</span><span class="p">(</span><span class="n">newsgroups_test</span><span class="o">.</span><span class="n">target</span><span class="p">,</span> <span class="n">pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.88213...</span>
</pre></div>
</div>
<p>(The example <a class="reference internal" href="../auto_examples/text/plot_document_classification_20newsgroups.html#sphx-glr-auto-examples-text-plot-document-classification-20newsgroups-py"><span class="std std-ref">Classification of text documents using sparse features</span></a> shuffles
the training and test data, instead of segmenting by time, and in that case
multinomial Naive Bayes gets a much higher F-score of 0.88. Are you suspicious
yet of what’s going on inside this classifier?)</p>
<p>Let’s take a look at what the most informative features are:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">show_top10</span><span class="p">(</span><span class="n">classifier</span><span class="p">,</span> <span class="n">vectorizer</span><span class="p">,</span> <span class="n">categories</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">feature_names</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">vectorizer</span><span class="o">.</span><span class="n">get_feature_names</span><span class="p">())</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">category</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">categories</span><span class="p">):</span>
<span class="gp">... </span>        <span class="n">top10</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argsort</span><span class="p">(</span><span class="n">classifier</span><span class="o">.</span><span class="n">coef_</span><span class="p">[</span><span class="n">i</span><span class="p">])[</span><span class="o">-</span><span class="mi">10</span><span class="p">:]</span>
<span class="gp">... </span>        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">category</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">feature_names</span><span class="p">[</span><span class="n">top10</span><span class="p">])))</span>
<span class="gp">...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">show_top10</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">vectorizer</span><span class="p">,</span> <span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">)</span>
<span class="go">alt.atheism: edu it and in you that is of to the</span>
<span class="go">comp.graphics: edu in graphics it is for and of to the</span>
<span class="go">sci.space: edu it that is in and space to of the</span>
<span class="go">talk.religion.misc: not it you in is that and to of the</span>
</pre></div>
</div>
<p>You can now see many things that these features have overfit to:</p>
<ul class="simple">
<li><p>Almost every group is distinguished by whether headers such as
<code class="docutils literal notranslate"><span class="pre">NNTP-Posting-Host:</span></code> and <code class="docutils literal notranslate"><span class="pre">Distribution:</span></code> appear more or less often.</p></li>
<li><p>Another significant feature involves whether the sender is affiliated with
a university, as indicated either by their headers or their signature.</p></li>
<li><p>The word “article” is a significant feature, based on how often people quote
previous posts like this: “In article [article ID], [name] &lt;[e-mail address]&gt;
wrote:”</p></li>
<li><p>Other features match the names and e-mail addresses of particular people who
were posting at the time.</p></li>
</ul>
<p>With such an abundance of clues that distinguish newsgroups, the classifiers
barely have to identify topics from text at all, and they all perform at the
same high level.</p>
<p>For this reason, the functions that load 20 Newsgroups data provide a
parameter called <strong>remove</strong>, telling it what kinds of information to strip out
of each file. <strong>remove</strong> should be a tuple containing any subset of
<code class="docutils literal notranslate"><span class="pre">('headers',</span> <span class="pre">'footers',</span> <span class="pre">'quotes')</span></code>, telling it to remove headers, signature
blocks, and quotation blocks respectively.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_test</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;test&#39;</span><span class="p">,</span>
<span class="gp">... </span>                                     <span class="n">remove</span><span class="o">=</span><span class="p">(</span><span class="s1">&#39;headers&#39;</span><span class="p">,</span> <span class="s1">&#39;footers&#39;</span><span class="p">,</span> <span class="s1">&#39;quotes&#39;</span><span class="p">),</span>
<span class="gp">... </span>                                     <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectors_test</span> <span class="o">=</span> <span class="n">vectorizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">newsgroups_test</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pred</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">vectors_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">f1_score</span><span class="p">(</span><span class="n">pred</span><span class="p">,</span> <span class="n">newsgroups_test</span><span class="o">.</span><span class="n">target</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.77310...</span>
</pre></div>
</div>
<p>This classifier lost over a lot of its F-score, just because we removed
metadata that has little to do with topic classification.
It loses even more if we also strip this metadata from the training data:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">newsgroups_train</span> <span class="o">=</span> <span class="n">fetch_20newsgroups</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">,</span>
<span class="gp">... </span>                                      <span class="n">remove</span><span class="o">=</span><span class="p">(</span><span class="s1">&#39;headers&#39;</span><span class="p">,</span> <span class="s1">&#39;footers&#39;</span><span class="p">,</span> <span class="s1">&#39;quotes&#39;</span><span class="p">),</span>
<span class="gp">... </span>                                      <span class="n">categories</span><span class="o">=</span><span class="n">categories</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">vectors</span> <span class="o">=</span> <span class="n">vectorizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">newsgroups_train</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">MultinomialNB</span><span class="p">(</span><span class="n">alpha</span><span class="o">=.</span><span class="mi">01</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">vectors</span><span class="p">,</span> <span class="n">newsgroups_train</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)</span>
</pre></div>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">vectors_test</span> <span class="o">=</span> <span class="n">vectorizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">newsgroups_test</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pred</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">vectors_test</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">f1_score</span><span class="p">(</span><span class="n">newsgroups_test</span><span class="o">.</span><span class="n">target</span><span class="p">,</span> <span class="n">pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.76995...</span>
</pre></div>
</div>
<p>Some other classifiers cope better with this harder version of the task. Try
running <a class="reference internal" href="../auto_examples/model_selection/grid_search_text_feature_extraction.html#sphx-glr-auto-examples-model-selection-grid-search-text-feature-extraction-py"><span class="std std-ref">Sample pipeline for text feature extraction and evaluation</span></a> with and without
the <code class="docutils literal notranslate"><span class="pre">--filter</span></code> option to compare the results.</p>
<div class="topic">
<p class="topic-title">Recommendation</p>
<p>When evaluating text classifiers on the 20 Newsgroups data, you
should strip newsgroup-related metadata. In scikit-learn, you can do this by
setting <code class="docutils literal notranslate"><span class="pre">remove=('headers',</span> <span class="pre">'footers',</span> <span class="pre">'quotes')</span></code>. The F-score will be
lower because it is more realistic.</p>
</div>
<div class="topic">
<p class="topic-title">Examples</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/model_selection/grid_search_text_feature_extraction.html#sphx-glr-auto-examples-model-selection-grid-search-text-feature-extraction-py"><span class="std std-ref">Sample pipeline for text feature extraction and evaluation</span></a></p></li>
<li><p><a class="reference internal" href="../auto_examples/text/plot_document_classification_20newsgroups.html#sphx-glr-auto-examples-text-plot-document-classification-20newsgroups-py"><span class="std std-ref">Classification of text documents using sparse features</span></a></p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="the-labeled-faces-in-the-wild-face-recognition-dataset">
<span id="labeled-faces-in-the-wild-dataset"></span><h3>7.3.3. The Labeled Faces in the Wild face recognition dataset<a class="headerlink" href="#the-labeled-faces-in-the-wild-face-recognition-dataset" title="Permalink to this headline">¶</a></h3>
<p>This dataset is a collection of JPEG pictures of famous people collected
over the internet, all details are available on the official website:</p>
<blockquote>
<div><p><a class="reference external" href="http://vis-www.cs.umass.edu/lfw/">http://vis-www.cs.umass.edu/lfw/</a></p>
</div></blockquote>
<p>Each picture is centered on a single face. The typical task is called
Face Verification: given a pair of two pictures, a binary classifier
must predict whether the two images are from the same person.</p>
<p>An alternative task, Face Recognition or Face Identification is:
given the picture of the face of an unknown person, identify the name
of the person by referring to a gallery of previously seen pictures of
identified persons.</p>
<p>Both Face Verification and Face Recognition are tasks that are typically
performed on the output of a model trained to perform Face Detection. The
most popular model for Face Detection is called Viola-Jones and is
implemented in the OpenCV library. The LFW faces were extracted by this
face detector from various online websites.</p>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 43%" />
<col style="width: 58%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Classes</p></td>
<td><p>5749</p></td>
</tr>
<tr class="row-even"><td><p>Samples total</p></td>
<td><p>13233</p></td>
</tr>
<tr class="row-odd"><td><p>Dimensionality</p></td>
<td><p>5828</p></td>
</tr>
<tr class="row-even"><td><p>Features</p></td>
<td><p>real, between 0 and 255</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<div class="section" id="id5">
<h4>7.3.3.1. Usage<a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h4>
<p><code class="docutils literal notranslate"><span class="pre">scikit-learn</span></code> provides two loaders that will automatically download,
cache, parse the metadata files, decode the jpeg and convert the
interesting slices into memmapped numpy arrays. This dataset size is more
than 200 MB. The first load typically takes more than a couple of minutes
to fully decode the relevant part of the JPEG files into numpy arrays. If
the dataset has  been loaded once, the following times the loading times
less than 200ms by using a memmapped version memoized on the disk in the
<code class="docutils literal notranslate"><span class="pre">~/scikit_learn_data/lfw_home/</span></code> folder using <code class="docutils literal notranslate"><span class="pre">joblib</span></code>.</p>
<p>The first loader is used for the Face Identification task: a multi-class
classification task (hence supervised learning):</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_lfw_people</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_people</span> <span class="o">=</span> <span class="n">fetch_lfw_people</span><span class="p">(</span><span class="n">min_faces_per_person</span><span class="o">=</span><span class="mi">70</span><span class="p">,</span> <span class="n">resize</span><span class="o">=</span><span class="mf">0.4</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">lfw_people</span><span class="o">.</span><span class="n">target_names</span><span class="p">:</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
<span class="gp">...</span>
<span class="go">Ariel Sharon</span>
<span class="go">Colin Powell</span>
<span class="go">Donald Rumsfeld</span>
<span class="go">George W Bush</span>
<span class="go">Gerhard Schroeder</span>
<span class="go">Hugo Chavez</span>
<span class="go">Tony Blair</span>
</pre></div>
</div>
<p>The default slice is a rectangular shape around the face, removing
most of the background:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_people</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">dtype</span>
<span class="go">dtype(&#39;float32&#39;)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_people</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1288, 1850)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_people</span><span class="o">.</span><span class="n">images</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1288, 50, 37)</span>
</pre></div>
</div>
<p>Each of the <code class="docutils literal notranslate"><span class="pre">1140</span></code> faces is assigned to a single person id in the <code class="docutils literal notranslate"><span class="pre">target</span></code>
array:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_people</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1288,)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="nb">list</span><span class="p">(</span><span class="n">lfw_people</span><span class="o">.</span><span class="n">target</span><span class="p">[:</span><span class="mi">10</span><span class="p">])</span>
<span class="go">[5, 6, 3, 1, 0, 1, 3, 4, 3, 0]</span>
</pre></div>
</div>
<p>The second loader is typically used for the face verification task: each sample
is a pair of two picture belonging or not to the same person:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_lfw_pairs</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_pairs_train</span> <span class="o">=</span> <span class="n">fetch_lfw_pairs</span><span class="p">(</span><span class="n">subset</span><span class="o">=</span><span class="s1">&#39;train&#39;</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="nb">list</span><span class="p">(</span><span class="n">lfw_pairs_train</span><span class="o">.</span><span class="n">target_names</span><span class="p">)</span>
<span class="go">[&#39;Different persons&#39;, &#39;Same person&#39;]</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_pairs_train</span><span class="o">.</span><span class="n">pairs</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2200, 2, 62, 47)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_pairs_train</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2200, 5828)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">lfw_pairs_train</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(2200,)</span>
</pre></div>
</div>
<p>Both for the <a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_lfw_people.html#sklearn.datasets.fetch_lfw_people" title="sklearn.datasets.fetch_lfw_people"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_lfw_people</span></code></a> and
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_lfw_pairs.html#sklearn.datasets.fetch_lfw_pairs" title="sklearn.datasets.fetch_lfw_pairs"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_lfw_pairs</span></code></a> function it is
possible to get an additional dimension with the RGB color channels by
passing <code class="docutils literal notranslate"><span class="pre">color=True</span></code>, in that case the shape will be
<code class="docutils literal notranslate"><span class="pre">(2200,</span> <span class="pre">2,</span> <span class="pre">62,</span> <span class="pre">47,</span> <span class="pre">3)</span></code>.</p>
<p>The <a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_lfw_pairs.html#sklearn.datasets.fetch_lfw_pairs" title="sklearn.datasets.fetch_lfw_pairs"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_lfw_pairs</span></code></a> datasets is subdivided into
3 subsets: the development <code class="docutils literal notranslate"><span class="pre">train</span></code> set, the development <code class="docutils literal notranslate"><span class="pre">test</span></code> set and
an evaluation <code class="docutils literal notranslate"><span class="pre">10_folds</span></code> set meant to compute performance metrics using a
10-folds cross validation scheme.</p>
<div class="topic">
<p class="topic-title">References:</p>
<ul class="simple">
<li><p><a class="reference external" href="http://vis-www.cs.umass.edu/lfw/lfw.pdf">Labeled Faces in the Wild: A Database for Studying Face Recognition
in Unconstrained Environments.</a>
Gary B. Huang, Manu Ramesh, Tamara Berg, and Erik Learned-Miller.
University of Massachusetts, Amherst, Technical Report 07-49, October, 2007.</p></li>
</ul>
</div>
</div>
<div class="section" id="examples">
<h4>7.3.3.2. Examples<a class="headerlink" href="#examples" title="Permalink to this headline">¶</a></h4>
<p><a class="reference internal" href="../auto_examples/applications/plot_face_recognition.html#sphx-glr-auto-examples-applications-plot-face-recognition-py"><span class="std std-ref">Faces recognition example using eigenfaces and SVMs</span></a></p>
</div>
</div>
<div class="section" id="forest-covertypes">
<span id="covtype-dataset"></span><h3>7.3.4. Forest covertypes<a class="headerlink" href="#forest-covertypes" title="Permalink to this headline">¶</a></h3>
<p>The samples in this dataset correspond to 30×30m patches of forest in the US,
collected for the task of predicting each patch’s cover type,
i.e. the dominant species of tree.
There are seven covertypes, making this a multiclass classification problem.
Each sample has 54 features, described on the
<a class="reference external" href="https://archive.ics.uci.edu/ml/datasets/Covertype">dataset’s homepage</a>.
Some of the features are boolean indicators,
while others are discrete or continuous measurements.</p>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 59%" />
<col style="width: 41%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Classes</p></td>
<td><p>7</p></td>
</tr>
<tr class="row-even"><td><p>Samples total</p></td>
<td><p>581012</p></td>
</tr>
<tr class="row-odd"><td><p>Dimensionality</p></td>
<td><p>54</p></td>
</tr>
<tr class="row-even"><td><p>Features</p></td>
<td><p>int</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_covtype.html#sklearn.datasets.fetch_covtype" title="sklearn.datasets.fetch_covtype"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_covtype</span></code></a> will load the covertype dataset;
it returns a dictionary-like object
with the feature matrix in the <code class="docutils literal notranslate"><span class="pre">data</span></code> member
and the target values in <code class="docutils literal notranslate"><span class="pre">target</span></code>.
The dataset will be downloaded from the web if necessary.</p>
</div>
<div class="section" id="rcv1-dataset">
<span id="id6"></span><h3>7.3.5. RCV1 dataset<a class="headerlink" href="#rcv1-dataset" title="Permalink to this headline">¶</a></h3>
<p>Reuters Corpus Volume I (RCV1) is an archive of over 800,000 manually
categorized newswire stories made available by Reuters, Ltd. for research
purposes. The dataset is extensively described in <a class="footnote-reference brackets" href="#id9" id="id7">1</a>.</p>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 40%" />
<col style="width: 60%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Classes</p></td>
<td><p>103</p></td>
</tr>
<tr class="row-even"><td><p>Samples total</p></td>
<td><p>804414</p></td>
</tr>
<tr class="row-odd"><td><p>Dimensionality</p></td>
<td><p>47236</p></td>
</tr>
<tr class="row-even"><td><p>Features</p></td>
<td><p>real, between 0 and 1</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_rcv1.html#sklearn.datasets.fetch_rcv1" title="sklearn.datasets.fetch_rcv1"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_rcv1</span></code></a> will load the following
version: RCV1-v2, vectors, full sets, topics multilabels:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_rcv1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">rcv1</span> <span class="o">=</span> <span class="n">fetch_rcv1</span><span class="p">()</span>
</pre></div>
</div>
<p>It returns a dictionary-like object, with the following attributes:</p>
<p><code class="docutils literal notranslate"><span class="pre">data</span></code>:
The feature matrix is a scipy CSR sparse matrix, with 804414 samples and
47236 features. Non-zero values contains cosine-normalized, log TF-IDF vectors.
A nearly chronological split is proposed in <a class="footnote-reference brackets" href="#id9" id="id8">1</a>: The first 23149 samples are
the training set. The last 781265 samples are the testing set. This follows
the official LYRL2004 chronological split. The array has 0.16% of non zero
values:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rcv1</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(804414, 47236)</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">target</span></code>:
The target values are stored in a scipy CSR sparse matrix, with 804414 samples
and 103 categories. Each sample has a value of 1 in its categories, and 0 in
others. The array has 3.15% of non zero values:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rcv1</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(804414, 103)</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">sample_id</span></code>:
Each sample can be identified by its ID, ranging (with gaps) from 2286
to 810596:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rcv1</span><span class="o">.</span><span class="n">sample_id</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span>
<span class="go">array([2286, 2287, 2288], dtype=uint32)</span>
</pre></div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">target_names</span></code>:
The target values are the topics of each sample. Each sample belongs to at
least one topic, and to up to 17 topics. There are 103 topics, each
represented by a string. Their corpus frequencies span five orders of
magnitude, from 5 occurrences for ‘GMIL’, to 381327 for ‘CCAT’:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rcv1</span><span class="o">.</span><span class="n">target_names</span><span class="p">[:</span><span class="mi">3</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>  
<span class="go">[&#39;E11&#39;, &#39;ECAT&#39;, &#39;M11&#39;]</span>
</pre></div>
</div>
<p>The dataset will be downloaded from the <a class="reference external" href="http://jmlr.csail.mit.edu/papers/volume5/lewis04a/">rcv1 homepage</a> if necessary.
The compressed size is about 656 MB.</p>
<div class="topic">
<p class="topic-title">References</p>
<dl class="footnote brackets">
<dt class="label" id="id9"><span class="brackets">1</span><span class="fn-backref">(<a href="#id7">1</a>,<a href="#id8">2</a>)</span></dt>
<dd><p>Lewis, D. D., Yang, Y., Rose, T. G., &amp; Li, F. (2004).
RCV1: A new benchmark collection for text categorization research.
The Journal of Machine Learning Research, 5, 361-397.</p>
</dd>
</dl>
</div>
</div>
<div class="section" id="kddcup-99-dataset">
<span id="kddcup99-dataset"></span><h3>7.3.6. Kddcup 99 dataset<a class="headerlink" href="#kddcup-99-dataset" title="Permalink to this headline">¶</a></h3>
<p>The KDD Cup ‘99 dataset was created by processing the tcpdump portions
of the 1998 DARPA Intrusion Detection System (IDS) Evaluation dataset,
created by MIT Lincoln Lab [1]. The artificial data (described on the <a class="reference external" href="https://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html">dataset’s
homepage</a>) was
generated using a closed network and hand-injected attacks to produce a
large number of different types of attack with normal activity in the
background. As the initial goal was to produce a large training set for
supervised learning algorithms, there is a large proportion (80.1%) of
abnormal data which is unrealistic in real world, and inappropriate for
unsupervised anomaly detection which aims at detecting ‘abnormal’ data, ie</p>
<ol class="arabic simple">
<li><p>qualitatively different from normal data</p></li>
<li><p>in large minority among the observations.</p></li>
</ol>
<p>We thus transform the KDD Data set into two different data sets: SA and SF.</p>
<p>-SA is obtained by simply selecting all the normal data, and a small
proportion of abnormal data to gives an anomaly proportion of 1%.</p>
<p>-SF is obtained as in [2]
by simply picking up the data whose attribute logged_in is positive, thus
focusing on the intrusion attack, which gives a proportion of 0.3% of
attack.</p>
<p>-http and smtp are two subsets of SF corresponding with third feature
equal to ‘http’ (resp. to ‘smtp’)</p>
<p>General KDD structure :</p>
<blockquote>
<div><table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 72%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Samples total</p></td>
<td><p>4898431</p></td>
</tr>
<tr class="row-even"><td><p>Dimensionality</p></td>
<td><p>41</p></td>
</tr>
<tr class="row-odd"><td><p>Features</p></td>
<td><p>discrete (int) or continuous (float)</p></td>
</tr>
<tr class="row-even"><td><p>Targets</p></td>
<td><p>str, ‘normal.’ or name of the anomaly type</p></td>
</tr>
</tbody>
</table>
<p>SA structure :</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 72%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Samples total</p></td>
<td><p>976158</p></td>
</tr>
<tr class="row-even"><td><p>Dimensionality</p></td>
<td><p>41</p></td>
</tr>
<tr class="row-odd"><td><p>Features</p></td>
<td><p>discrete (int) or continuous (float)</p></td>
</tr>
<tr class="row-even"><td><p>Targets</p></td>
<td><p>str, ‘normal.’ or name of the anomaly type</p></td>
</tr>
</tbody>
</table>
<p>SF structure :</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 72%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Samples total</p></td>
<td><p>699691</p></td>
</tr>
<tr class="row-even"><td><p>Dimensionality</p></td>
<td><p>4</p></td>
</tr>
<tr class="row-odd"><td><p>Features</p></td>
<td><p>discrete (int) or continuous (float)</p></td>
</tr>
<tr class="row-even"><td><p>Targets</p></td>
<td><p>str, ‘normal.’ or name of the anomaly type</p></td>
</tr>
</tbody>
</table>
<p>http structure :</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 72%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Samples total</p></td>
<td><p>619052</p></td>
</tr>
<tr class="row-even"><td><p>Dimensionality</p></td>
<td><p>3</p></td>
</tr>
<tr class="row-odd"><td><p>Features</p></td>
<td><p>discrete (int) or continuous (float)</p></td>
</tr>
<tr class="row-even"><td><p>Targets</p></td>
<td><p>str, ‘normal.’ or name of the anomaly type</p></td>
</tr>
</tbody>
</table>
<p>smtp structure :</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 72%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p>Samples total</p></td>
<td><p>95373</p></td>
</tr>
<tr class="row-even"><td><p>Dimensionality</p></td>
<td><p>3</p></td>
</tr>
<tr class="row-odd"><td><p>Features</p></td>
<td><p>discrete (int) or continuous (float)</p></td>
</tr>
<tr class="row-even"><td><p>Targets</p></td>
<td><p>str, ‘normal.’ or name of the anomaly type</p></td>
</tr>
</tbody>
</table>
</div></blockquote>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_kddcup99.html#sklearn.datasets.fetch_kddcup99" title="sklearn.datasets.fetch_kddcup99"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_kddcup99</span></code></a> will load the kddcup99 dataset; it
returns a dictionary-like object with the feature matrix in the <code class="docutils literal notranslate"><span class="pre">data</span></code> member
and the target values in <code class="docutils literal notranslate"><span class="pre">target</span></code>. The dataset will be downloaded from the
web if necessary.</p>
</div>
<div class="section" id="california-housing-dataset">
<span id="id10"></span><h3>7.3.7. California Housing dataset<a class="headerlink" href="#california-housing-dataset" title="Permalink to this headline">¶</a></h3>
<p><strong>Data Set Characteristics:</strong></p>
<blockquote>
<div><dl class="field-list simple">
<dt class="field-odd">Number of Instances</dt>
<dd class="field-odd"><p>20640</p>
</dd>
<dt class="field-even">Number of Attributes</dt>
<dd class="field-even"><p>8 numeric, predictive attributes and the target</p>
</dd>
<dt class="field-odd">Attribute Information</dt>
<dd class="field-odd"><ul class="simple">
<li><p>MedInc        median income in block</p></li>
<li><p>HouseAge      median house age in block</p></li>
<li><p>AveRooms      average number of rooms</p></li>
<li><p>AveBedrms     average number of bedrooms</p></li>
<li><p>Population    block population</p></li>
<li><p>AveOccup      average house occupancy</p></li>
<li><p>Latitude      house block latitude</p></li>
<li><p>Longitude     house block longitude</p></li>
</ul>
</dd>
<dt class="field-even">Missing Attribute Values</dt>
<dd class="field-even"><p>None</p>
</dd>
</dl>
</div></blockquote>
<p>This dataset was obtained from the StatLib repository.
<a class="reference external" href="http://lib.stat.cmu.edu/datasets/">http://lib.stat.cmu.edu/datasets/</a></p>
<p>The target variable is the median house value for California districts.</p>
<p>This dataset was derived from the 1990 U.S. census, using one row per census
block group. A block group is the smallest geographical unit for which the U.S.
Census Bureau publishes sample data (a block group typically has a population
of 600 to 3,000 people).</p>
<p>It can be downloaded/loaded using the
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_california_housing.html#sklearn.datasets.fetch_california_housing" title="sklearn.datasets.fetch_california_housing"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_california_housing</span></code></a> function.</p>
<div class="topic">
<p class="topic-title">References</p>
<ul class="simple">
<li><p>Pace, R. Kelley and Ronald Barry, Sparse Spatial Autoregressions,
Statistics and Probability Letters, 33 (1997) 291-297</p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="generated-datasets">
<span id="sample-generators"></span><h2>7.4. Generated datasets<a class="headerlink" href="#generated-datasets" title="Permalink to this headline">¶</a></h2>
<p>In addition, scikit-learn includes various random sample generators that
can be used to build artificial datasets of controlled size and complexity.</p>
<div class="section" id="generators-for-classification-and-clustering">
<h3>7.4.1. Generators for classification and clustering<a class="headerlink" href="#generators-for-classification-and-clustering" title="Permalink to this headline">¶</a></h3>
<p>These generators produce a matrix of features and corresponding discrete
targets.</p>
<div class="section" id="single-label">
<h4>7.4.1.1. Single label<a class="headerlink" href="#single-label" title="Permalink to this headline">¶</a></h4>
<p>Both <a class="reference internal" href="../modules/generated/sklearn.datasets.make_blobs.html#sklearn.datasets.make_blobs" title="sklearn.datasets.make_blobs"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_blobs</span></code></a> and <a class="reference internal" href="../modules/generated/sklearn.datasets.make_classification.html#sklearn.datasets.make_classification" title="sklearn.datasets.make_classification"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_classification</span></code></a> create multiclass
datasets by allocating each class one or more normally-distributed clusters of
points.  <a class="reference internal" href="../modules/generated/sklearn.datasets.make_blobs.html#sklearn.datasets.make_blobs" title="sklearn.datasets.make_blobs"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_blobs</span></code></a> provides greater control regarding the centers and
standard deviations of each cluster, and is used to demonstrate clustering.
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_classification.html#sklearn.datasets.make_classification" title="sklearn.datasets.make_classification"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_classification</span></code></a> specialises in introducing noise by way of:
correlated, redundant and uninformative features; multiple Gaussian clusters
per class; and linear transformations of the feature space.</p>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_gaussian_quantiles.html#sklearn.datasets.make_gaussian_quantiles" title="sklearn.datasets.make_gaussian_quantiles"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_gaussian_quantiles</span></code></a> divides a single Gaussian cluster into
near-equal-size classes separated by concentric hyperspheres.
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_hastie_10_2.html#sklearn.datasets.make_hastie_10_2" title="sklearn.datasets.make_hastie_10_2"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_hastie_10_2</span></code></a> generates a similar binary, 10-dimensional problem.</p>
<a class="reference external image-reference" href="../auto_examples/datasets/plot_random_dataset.html"><img alt="datasets/../auto_examples/datasets/images/sphx_glr_plot_random_dataset_001.png" class="align-center" src="datasets/../auto_examples/datasets/images/sphx_glr_plot_random_dataset_001.png" /></a>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_circles.html#sklearn.datasets.make_circles" title="sklearn.datasets.make_circles"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_circles</span></code></a> and <a class="reference internal" href="../modules/generated/sklearn.datasets.make_moons.html#sklearn.datasets.make_moons" title="sklearn.datasets.make_moons"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_moons</span></code></a> generate 2d binary classification
datasets that are challenging to certain algorithms (e.g. centroid-based
clustering or linear classification), including optional Gaussian noise.
They are useful for visualisation. <a class="reference internal" href="../modules/generated/sklearn.datasets.make_circles.html#sklearn.datasets.make_circles" title="sklearn.datasets.make_circles"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_circles</span></code></a> produces Gaussian data
with a spherical decision boundary for binary classification, while
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_moons.html#sklearn.datasets.make_moons" title="sklearn.datasets.make_moons"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_moons</span></code></a> produces two interleaving half circles.</p>
</div>
<div class="section" id="multilabel">
<h4>7.4.1.2. Multilabel<a class="headerlink" href="#multilabel" title="Permalink to this headline">¶</a></h4>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_multilabel_classification.html#sklearn.datasets.make_multilabel_classification" title="sklearn.datasets.make_multilabel_classification"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_multilabel_classification</span></code></a> generates random samples with multiple
labels, reflecting a bag of words drawn from a mixture of topics. The number of
topics for each document is drawn from a Poisson distribution, and the topics
themselves are drawn from a fixed random distribution. Similarly, the number of
words is drawn from Poisson, with words drawn from a multinomial, where each
topic defines a probability distribution over words. Simplifications with
respect to true bag-of-words mixtures include:</p>
<ul class="simple">
<li><p>Per-topic word distributions are independently drawn, where in reality all
would be affected by a sparse base distribution, and would be correlated.</p></li>
<li><p>For a document generated from multiple topics, all topics are weighted
equally in generating its bag of words.</p></li>
<li><p>Documents without labels words at random, rather than from a base
distribution.</p></li>
</ul>
<a class="reference external image-reference" href="../auto_examples/datasets/plot_random_multilabel_dataset.html"><img alt="datasets/../auto_examples/datasets/images/sphx_glr_plot_random_multilabel_dataset_001.png" class="align-center" src="datasets/../auto_examples/datasets/images/sphx_glr_plot_random_multilabel_dataset_001.png" /></a>
</div>
<div class="section" id="biclustering">
<h4>7.4.1.3. Biclustering<a class="headerlink" href="#biclustering" title="Permalink to this headline">¶</a></h4>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_biclusters.html#sklearn.datasets.make_biclusters" title="sklearn.datasets.make_biclusters"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_biclusters</span></code></a>(shape, n_clusters[, noise, …])</p></td>
<td><p>Generate an array with constant block diagonal structure for biclustering.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_checkerboard.html#sklearn.datasets.make_checkerboard" title="sklearn.datasets.make_checkerboard"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_checkerboard</span></code></a>(shape, n_clusters[, …])</p></td>
<td><p>Generate an array with block checkerboard structure for biclustering.</p></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="section" id="generators-for-regression">
<h3>7.4.2. Generators for regression<a class="headerlink" href="#generators-for-regression" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_regression.html#sklearn.datasets.make_regression" title="sklearn.datasets.make_regression"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_regression</span></code></a> produces regression targets as an optionally-sparse
random linear combination of random features, with noise. Its informative
features may be uncorrelated, or low rank (few features account for most of the
variance).</p>
<p>Other regression generators generate functions deterministically from
randomized features.  <a class="reference internal" href="../modules/generated/sklearn.datasets.make_sparse_uncorrelated.html#sklearn.datasets.make_sparse_uncorrelated" title="sklearn.datasets.make_sparse_uncorrelated"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_sparse_uncorrelated</span></code></a> produces a target as a
linear combination of four features with fixed coefficients.
Others encode explicitly non-linear relations:
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_friedman1.html#sklearn.datasets.make_friedman1" title="sklearn.datasets.make_friedman1"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_friedman1</span></code></a> is related by polynomial and sine transforms;
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_friedman2.html#sklearn.datasets.make_friedman2" title="sklearn.datasets.make_friedman2"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_friedman2</span></code></a> includes feature multiplication and reciprocation; and
<a class="reference internal" href="../modules/generated/sklearn.datasets.make_friedman3.html#sklearn.datasets.make_friedman3" title="sklearn.datasets.make_friedman3"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_friedman3</span></code></a> is similar with an arctan transformation on the target.</p>
</div>
<div class="section" id="generators-for-manifold-learning">
<h3>7.4.3. Generators for manifold learning<a class="headerlink" href="#generators-for-manifold-learning" title="Permalink to this headline">¶</a></h3>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_s_curve.html#sklearn.datasets.make_s_curve" title="sklearn.datasets.make_s_curve"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_s_curve</span></code></a>([n_samples, noise, random_state])</p></td>
<td><p>Generate an S curve dataset.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_swiss_roll.html#sklearn.datasets.make_swiss_roll" title="sklearn.datasets.make_swiss_roll"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_swiss_roll</span></code></a>([n_samples, noise, random_state])</p></td>
<td><p>Generate a swiss roll dataset.</p></td>
</tr>
</tbody>
</table>
</div>
<div class="section" id="generators-for-decomposition">
<h3>7.4.4. Generators for decomposition<a class="headerlink" href="#generators-for-decomposition" title="Permalink to this headline">¶</a></h3>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_low_rank_matrix.html#sklearn.datasets.make_low_rank_matrix" title="sklearn.datasets.make_low_rank_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_low_rank_matrix</span></code></a>([n_samples, …])</p></td>
<td><p>Generate a mostly low rank matrix with bell-shaped singular values</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_sparse_coded_signal.html#sklearn.datasets.make_sparse_coded_signal" title="sklearn.datasets.make_sparse_coded_signal"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_sparse_coded_signal</span></code></a>(n_samples, …[, …])</p></td>
<td><p>Generate a signal as a sparse combination of dictionary elements.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_spd_matrix.html#sklearn.datasets.make_spd_matrix" title="sklearn.datasets.make_spd_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_spd_matrix</span></code></a>(n_dim[, random_state])</p></td>
<td><p>Generate a random symmetric, positive-definite matrix.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.make_sparse_spd_matrix.html#sklearn.datasets.make_sparse_spd_matrix" title="sklearn.datasets.make_sparse_spd_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">make_sparse_spd_matrix</span></code></a>([dim, alpha, …])</p></td>
<td><p>Generate a sparse symmetric definite positive matrix.</p></td>
</tr>
</tbody>
</table>
</div>
</div>
<div class="section" id="loading-other-datasets">
<span id="id11"></span><h2>7.5. Loading other datasets<a class="headerlink" href="#loading-other-datasets" title="Permalink to this headline">¶</a></h2>
<div class="section" id="sample-images">
<span id="id12"></span><h3>7.5.1. Sample images<a class="headerlink" href="#sample-images" title="Permalink to this headline">¶</a></h3>
<p>Scikit-learn also embed a couple of sample JPEG images published under Creative
Commons license by their authors. Those images can be useful to test algorithms
and pipeline on 2D data.</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_sample_images.html#sklearn.datasets.load_sample_images" title="sklearn.datasets.load_sample_images"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_sample_images</span></code></a>()</p></td>
<td><p>Load sample images for image manipulation.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="../modules/generated/sklearn.datasets.load_sample_image.html#sklearn.datasets.load_sample_image" title="sklearn.datasets.load_sample_image"><code class="xref py py-obj docutils literal notranslate"><span class="pre">load_sample_image</span></code></a>(image_name)</p></td>
<td><p>Load the numpy array of a single sample image</p></td>
</tr>
</tbody>
</table>
<a class="reference external image-reference" href="../auto_examples/cluster/plot_color_quantization.html"><img alt="datasets/../auto_examples/cluster/images/sphx_glr_plot_color_quantization_001.png" class="align-right" src="datasets/../auto_examples/cluster/images/sphx_glr_plot_color_quantization_001.png" /></a>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The default coding of images is based on the <code class="docutils literal notranslate"><span class="pre">uint8</span></code> dtype to
spare memory.  Often machine learning algorithms work best if the
input is converted to a floating point representation first.  Also,
if you plan to use <code class="docutils literal notranslate"><span class="pre">matplotlib.pyplpt.imshow</span></code> don’t forget to scale to the range
0 - 1 as done in the following example.</p>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p><a class="reference internal" href="../auto_examples/cluster/plot_color_quantization.html#sphx-glr-auto-examples-cluster-plot-color-quantization-py"><span class="std std-ref">Color Quantization using K-Means</span></a></p></li>
</ul>
</div>
</div>
<div class="section" id="datasets-in-svmlight-libsvm-format">
<span id="libsvm-loader"></span><h3>7.5.2. Datasets in svmlight / libsvm format<a class="headerlink" href="#datasets-in-svmlight-libsvm-format" title="Permalink to this headline">¶</a></h3>
<p>scikit-learn includes utility functions for loading
datasets in the svmlight / libsvm format. In this format, each line
takes the form <code class="docutils literal notranslate"><span class="pre">&lt;label&gt;</span> <span class="pre">&lt;feature-id&gt;:&lt;feature-value&gt;</span>
<span class="pre">&lt;feature-id&gt;:&lt;feature-value&gt;</span> <span class="pre">...</span></code>. This format is especially suitable for sparse datasets.
In this module, scipy sparse CSR matrices are used for <code class="docutils literal notranslate"><span class="pre">X</span></code> and numpy arrays are used for <code class="docutils literal notranslate"><span class="pre">y</span></code>.</p>
<p>You may load a dataset like as follows:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_svmlight_file</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span> <span class="o">=</span> <span class="n">load_svmlight_file</span><span class="p">(</span><span class="s2">&quot;/path/to/train_dataset.txt&quot;</span><span class="p">)</span>
<span class="gp">... </span>                                                        
</pre></div>
</div>
<p>You may also load two (or more) datasets at once:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">load_svmlight_files</span><span class="p">(</span>
<span class="gp">... </span>    <span class="p">(</span><span class="s2">&quot;/path/to/train_dataset.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;/path/to/test_dataset.txt&quot;</span><span class="p">))</span>
<span class="gp">... </span>                                                        
</pre></div>
</div>
<p>In this case, <code class="docutils literal notranslate"><span class="pre">X_train</span></code> and <code class="docutils literal notranslate"><span class="pre">X_test</span></code> are guaranteed to have the same number
of features. Another way to achieve the same result is to fix the number of
features:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">load_svmlight_file</span><span class="p">(</span>
<span class="gp">... </span>    <span class="s2">&quot;/path/to/test_dataset.txt&quot;</span><span class="p">,</span> <span class="n">n_features</span><span class="o">=</span><span class="n">X_train</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
<span class="gp">... </span>                                                        
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Related links:</p>
<p><span class="target" id="public-datasets-in-svmlight-libsvm-format">Public datasets in svmlight / libsvm format</span>: <a class="reference external" href="https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets">https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets</a></p>
<p><span class="target" id="faster-api-compatible-implementation">Faster API-compatible implementation</span>: <a class="reference external" href="https://github.com/mblondel/svmlight-loader">https://github.com/mblondel/svmlight-loader</a></p>
</div>
</div>
<div class="section" id="downloading-datasets-from-the-openml-org-repository">
<span id="openml"></span><h3>7.5.3. Downloading datasets from the openml.org repository<a class="headerlink" href="#downloading-datasets-from-the-openml-org-repository" title="Permalink to this headline">¶</a></h3>
<p><a class="reference external" href="https://openml.org">openml.org</a> is a public repository for machine learning
data and experiments, that allows everybody to upload open datasets.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">sklearn.datasets</span></code> package is able to download datasets
from the repository using the function
<a class="reference internal" href="../modules/generated/sklearn.datasets.fetch_openml.html#sklearn.datasets.fetch_openml" title="sklearn.datasets.fetch_openml"><code class="xref py py-func docutils literal notranslate"><span class="pre">sklearn.datasets.fetch_openml</span></code></a>.</p>
<p>For example, to download a dataset of gene expressions in mice brains:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">fetch_openml</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s1">&#39;miceprotein&#39;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
</pre></div>
</div>
<p>To fully specify a dataset, you need to provide a name and a version, though
the version is optional, see <a class="reference internal" href="#openml-versions"><span class="std std-ref">Dataset Versions</span></a> below.
The dataset contains a total of 1080 examples belonging to 8 different
classes:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1080, 77)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">shape</span>
<span class="go">(1080,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">mice</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">array([&#39;c-CS-m&#39;, &#39;c-CS-s&#39;, &#39;c-SC-m&#39;, &#39;c-SC-s&#39;, &#39;t-CS-m&#39;, &#39;t-CS-s&#39;, &#39;t-SC-m&#39;, &#39;t-SC-s&#39;], dtype=object)</span>
</pre></div>
</div>
<p>You can get more information on the dataset by looking at the <code class="docutils literal notranslate"><span class="pre">DESCR</span></code>
and <code class="docutils literal notranslate"><span class="pre">details</span></code> attributes:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">mice</span><span class="o">.</span><span class="n">DESCR</span><span class="p">)</span> 
<span class="go">**Author**: Clara Higuera, Katheleen J. Gardiner, Krzysztof J. Cios</span>
<span class="go">**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/Mice+Protein+Expression) - 2015</span>
<span class="go">**Please cite**: Higuera C, Gardiner KJ, Cios KJ (2015) Self-Organizing</span>
<span class="go">Feature Maps Identify Proteins Critical to Learning in a Mouse Model of Down</span>
<span class="go">Syndrome. PLoS ONE 10(6): e0129126...</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">details</span> 
<span class="go">{&#39;id&#39;: &#39;40966&#39;, &#39;name&#39;: &#39;MiceProtein&#39;, &#39;version&#39;: &#39;4&#39;, &#39;format&#39;: &#39;ARFF&#39;,</span>
<span class="go">&#39;upload_date&#39;: &#39;2017-11-08T16:00:15&#39;, &#39;licence&#39;: &#39;Public&#39;,</span>
<span class="go">&#39;url&#39;: &#39;https://www.openml.org/data/v1/download/17928620/MiceProtein.arff&#39;,</span>
<span class="go">&#39;file_id&#39;: &#39;17928620&#39;, &#39;default_target_attribute&#39;: &#39;class&#39;,</span>
<span class="go">&#39;row_id_attribute&#39;: &#39;MouseID&#39;,</span>
<span class="go">&#39;ignore_attribute&#39;: [&#39;Genotype&#39;, &#39;Treatment&#39;, &#39;Behavior&#39;],</span>
<span class="go">&#39;tag&#39;: [&#39;OpenML-CC18&#39;, &#39;study_135&#39;, &#39;study_98&#39;, &#39;study_99&#39;],</span>
<span class="go">&#39;visibility&#39;: &#39;public&#39;, &#39;status&#39;: &#39;active&#39;,</span>
<span class="go">&#39;md5_checksum&#39;: &#39;3c479a6885bfa0438971388283a1ce32&#39;}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">DESCR</span></code> contains a free-text description of the data, while <code class="docutils literal notranslate"><span class="pre">details</span></code>
contains a dictionary of meta-data stored by openml, like the dataset id.
For more details, see the <a class="reference external" href="https://docs.openml.org/#data">OpenML documentation</a> The <code class="docutils literal notranslate"><span class="pre">data_id</span></code> of the mice protein dataset
is 40966, and you can use this (or the name) to get more information on the
dataset on the openml website:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">url</span>
<span class="go">&#39;https://www.openml.org/d/40966&#39;</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">data_id</span></code> also uniquely identifies a dataset from OpenML:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">data_id</span><span class="o">=</span><span class="mi">40966</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">details</span> 
<span class="go">{&#39;id&#39;: &#39;4550&#39;, &#39;name&#39;: &#39;MiceProtein&#39;, &#39;version&#39;: &#39;1&#39;, &#39;format&#39;: &#39;ARFF&#39;,</span>
<span class="go">&#39;creator&#39;: ...,</span>
<span class="go">&#39;upload_date&#39;: &#39;2016-02-17T14:32:49&#39;, &#39;licence&#39;: &#39;Public&#39;, &#39;url&#39;:</span>
<span class="go">&#39;https://www.openml.org/data/v1/download/1804243/MiceProtein.ARFF&#39;, &#39;file_id&#39;:</span>
<span class="go">&#39;1804243&#39;, &#39;default_target_attribute&#39;: &#39;class&#39;, &#39;citation&#39;: &#39;Higuera C,</span>
<span class="go">Gardiner KJ, Cios KJ (2015) Self-Organizing Feature Maps Identify Proteins</span>
<span class="go">Critical to Learning in a Mouse Model of Down Syndrome. PLoS ONE 10(6):</span>
<span class="go">e0129126. [Web Link] journal.pone.0129126&#39;, &#39;tag&#39;: [&#39;OpenML100&#39;, &#39;study_14&#39;,</span>
<span class="go">&#39;study_34&#39;], &#39;visibility&#39;: &#39;public&#39;, &#39;status&#39;: &#39;active&#39;, &#39;md5_checksum&#39;:</span>
<span class="go">&#39;3c479a6885bfa0438971388283a1ce32&#39;}</span>
</pre></div>
</div>
<div class="section" id="dataset-versions">
<span id="openml-versions"></span><h4>7.5.3.1. Dataset Versions<a class="headerlink" href="#dataset-versions" title="Permalink to this headline">¶</a></h4>
<p>A dataset is uniquely specified by its <code class="docutils literal notranslate"><span class="pre">data_id</span></code>, but not necessarily by its
name. Several different “versions” of a dataset with the same name can exist
which can contain entirely different datasets.
If a particular version of a dataset has been found to contain significant
issues, it might be deactivated. Using a name to specify a dataset will yield
the earliest version of a dataset that is still active. That means that
<code class="docutils literal notranslate"><span class="pre">fetch_openml(name=&quot;miceprotein&quot;)</span></code> can yield different results at different
times if earlier versions become inactive.
You can see that the dataset with <code class="docutils literal notranslate"><span class="pre">data_id</span></code> 40966 that we fetched above is
the version 1 of the “miceprotein” dataset:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mice</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span>  
<span class="go">&#39;1&#39;</span>
</pre></div>
</div>
<p>In fact, this dataset only has one version. The iris dataset on the other hand
has multiple versions:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">iris</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;iris&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span>  
<span class="go">&#39;1&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span>  
<span class="go">&#39;61&#39;</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_61</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">data_id</span><span class="o">=</span><span class="mi">61</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_61</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span>
<span class="go">&#39;1&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_61</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span>
<span class="go">&#39;61&#39;</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_969</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">data_id</span><span class="o">=</span><span class="mi">969</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_969</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span>
<span class="go">&#39;3&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_969</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span>
<span class="go">&#39;969&#39;</span>
</pre></div>
</div>
<p>Specifying the dataset by the name “iris” yields the lowest version, version 1,
with the <code class="docutils literal notranslate"><span class="pre">data_id</span></code> 61. To make sure you always get this exact dataset, it is
safest to specify it by the dataset <code class="docutils literal notranslate"><span class="pre">data_id</span></code>. The other dataset, with
<code class="docutils literal notranslate"><span class="pre">data_id</span></code> 969, is version 3 (version 2 has become inactive), and contains a
binarized version of the data:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">np</span><span class="o">.</span><span class="n">unique</span><span class="p">(</span><span class="n">iris_969</span><span class="o">.</span><span class="n">target</span><span class="p">)</span>
<span class="go">array([&#39;N&#39;, &#39;P&#39;], dtype=object)</span>
</pre></div>
</div>
<p>You can also specify both the name and the version, which also uniquely
identifies the dataset:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">iris_version_3</span> <span class="o">=</span> <span class="n">fetch_openml</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="s2">&quot;iris&quot;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_version_3</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;version&#39;</span><span class="p">]</span>
<span class="go">&#39;3&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iris_version_3</span><span class="o">.</span><span class="n">details</span><span class="p">[</span><span class="s1">&#39;id&#39;</span><span class="p">]</span>
<span class="go">&#39;969&#39;</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<ul class="simple">
<li><p>Vanschoren, van Rijn, Bischl and Torgo
<a class="reference external" href="https://arxiv.org/pdf/1407.7722.pdf">“OpenML: networked science in machine learning”</a>,
ACM SIGKDD Explorations Newsletter, 15(2), 49-60, 2014.</p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="loading-from-external-datasets">
<span id="external-datasets"></span><h3>7.5.4. Loading from external datasets<a class="headerlink" href="#loading-from-external-datasets" title="Permalink to this headline">¶</a></h3>
<p>scikit-learn works on any numeric data stored as numpy arrays or scipy sparse
matrices. Other types that are convertible to numeric arrays such as pandas
DataFrame are also acceptable.</p>
<p>Here are some recommended ways to load standard columnar data into a
format usable by scikit-learn:</p>
<ul class="simple">
<li><p><a class="reference external" href="https://pandas.pydata.org/pandas-docs/stable/io.html">pandas.io</a>
provides tools to read data from common formats including CSV, Excel, JSON
and SQL. DataFrames may also be constructed from lists of tuples or dicts.
Pandas handles heterogeneous data smoothly and provides tools for
manipulation and conversion into a numeric array suitable for scikit-learn.</p></li>
<li><p><a class="reference external" href="https://docs.scipy.org/doc/scipy/reference/io.html">scipy.io</a>
specializes in binary formats often used in scientific computing
context such as .mat and .arff</p></li>
<li><p><a class="reference external" href="https://docs.scipy.org/doc/numpy/reference/routines.io.html">numpy/routines.io</a>
for standard loading of columnar data into numpy arrays</p></li>
<li><p>scikit-learn’s <code class="xref py py-func docutils literal notranslate"><span class="pre">datasets.load_svmlight_file</span></code> for the svmlight or libSVM
sparse format</p></li>
<li><p>scikit-learn’s <code class="xref py py-func docutils literal notranslate"><span class="pre">datasets.load_files</span></code> for directories of text files where
the name of each directory is the name of each category and each file inside
of each directory corresponds to one sample from that category</p></li>
</ul>
<p>For some miscellaneous data such as images, videos, and audio, you may wish to
refer to:</p>
<ul class="simple">
<li><p><a class="reference external" href="https://scikit-image.org/docs/dev/api/skimage.io.html">skimage.io</a> or
<a class="reference external" href="https://imageio.readthedocs.io/en/latest/userapi.html">Imageio</a>
for loading images and videos into numpy arrays</p></li>
<li><p><a class="reference external" href="https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.io.wavfile.read.html">scipy.io.wavfile.read</a>
for reading WAV files into a numpy array</p></li>
</ul>
<p>Categorical (or nominal) features stored as strings (common in pandas DataFrames)
will need converting to numerical features using <a class="reference internal" href="../modules/generated/sklearn.preprocessing.OneHotEncoder.html#sklearn.preprocessing.OneHotEncoder" title="sklearn.preprocessing.OneHotEncoder"><code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn.preprocessing.OneHotEncoder</span></code></a>
or <a class="reference internal" href="../modules/generated/sklearn.preprocessing.OrdinalEncoder.html#sklearn.preprocessing.OrdinalEncoder" title="sklearn.preprocessing.OrdinalEncoder"><code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn.preprocessing.OrdinalEncoder</span></code></a> or similar.
See <a class="reference internal" href="../modules/preprocessing.html#preprocessing"><span class="std std-ref">Preprocessing data</span></a>.</p>
<p>Note: if you manage your own numerical data it is recommended to use an
optimized file format such as HDF5 to reduce data load times. Various libraries
such as H5Py, PyTables and pandas provides a Python interface for reading and
writing data in that format.</p>
</div>
</div>
</div>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2019, scikit-learn developers (BSD License).
          <a href="../_sources/datasets/index.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');
    var hide_text = 'Hide prompts and outputs';
    var show_text = 'Show prompts and outputs';

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        if (jthis.find('.gp').length > 0) {
            var button = $('<span class="copybutton">&gt;&gt;&gt;</span>');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
            jthis.prepend(button);
        }
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

    // define the behavior of the button when it's clicked
    $('.copybutton').click(function(e){
        e.preventDefault();
        var button = $(this);
        if (button.data('hidden') === 'false') {
            // hide the code output
            button.parent().find('.go, .gp, .gt').hide();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
            button.css('text-decoration', 'line-through');
            button.attr('title', show_text);
            button.data('hidden', 'true');
        } else {
            // show the code output
            button.parent().find('.go, .gp, .gt').show();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
            button.css('text-decoration', 'none');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
        }
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
  /*** Hide navbar when scrolling down ***/
  // Returns true when headerlink target matches hash in url
  (function() {
    hashTargetOnTop = function() {
        var hash = window.location.hash;
        if ( hash.length < 2 ) { return false; }

        var target = document.getElementById( hash.slice(1) );
        if ( target === null ) { return false; }

        var top = target.getBoundingClientRect().top;
        return (top < 2) && (top > -2);
    };

    // Hide navbar on load if hash target is on top
    var navBar = document.getElementById("navbar");
    var navBarToggler = document.getElementById("sk-navbar-toggler");
    var navBarHeightHidden = "-" + navBar.getBoundingClientRect().height + "px";
    var $window = $(window);

    hideNavBar = function() {
        navBar.style.top = navBarHeightHidden;
    };

    showNavBar = function() {
        navBar.style.top = "0";
    }

    if (hashTargetOnTop()) {
        hideNavBar()
    }

    var prevScrollpos = window.pageYOffset;
    hideOnScroll = function(lastScrollTop) {
        if (($window.width() < 768) && (navBarToggler.getAttribute("aria-expanded") === 'true')) {
            return;
        }
        if (lastScrollTop > 2 && (prevScrollpos <= lastScrollTop) || hashTargetOnTop()){
            hideNavBar()
        } else {
            showNavBar()
        }
        prevScrollpos = lastScrollTop;
    };

    /*** high preformance scroll event listener***/
    var raf = window.requestAnimationFrame ||
        window.webkitRequestAnimationFrame ||
        window.mozRequestAnimationFrame ||
        window.msRequestAnimationFrame ||
        window.oRequestAnimationFrame;
    var lastScrollTop = $window.scrollTop();

    if (raf) {
        loop();
    }

    function loop() {
        var scrollTop = $window.scrollTop();
        if (lastScrollTop === scrollTop) {
            raf(loop);
            return;
        } else {
            lastScrollTop = scrollTop;
            hideOnScroll(lastScrollTop);
            raf(loop);
        }
    }
  })();
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
</body>
</html>